Skip to content

Commit

Permalink
Added KoralQuery check when updating VC (solved #676)
Browse files Browse the repository at this point in the history 
Change-Id: Ic7f8f565fde6c4a5bf087e27af3f47674854a0f6
  • Loading branch information
@margaretha
margaretha committed Apr 10, 2024
1 parent cd12b37 commit 4478956
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 38 deletions.
2 changes: 1 addition & 1 deletion Changes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
- Enables inputting free-resources.json from data folder
- Changed loading external kustvakt.conf and jdbc.properties
to use /data folder (#598)

- Added KoralQuery check when updating VC (solved #676)


# version 0.73
Expand Down
112 changes: 80 additions & 32 deletions src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import de.ids_mannheim.korap.cache.VirtualCorpusCache;
import de.ids_mannheim.korap.constant.QueryType;
import de.ids_mannheim.korap.constant.ResourceType;
import de.ids_mannheim.korap.entity.QueryDO;
import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.exceptions.StatusCodes;
import de.ids_mannheim.korap.service.QueryService;
Expand Down Expand Up @@ -64,6 +65,11 @@ public void run () {
}
}

public void loadVCToCache (String filename, String filePath)
throws IOException, QueryException, KustvaktException {
loadVCToCache(filename,filePath,null);
}

/**
* Used for testing
*
Expand All @@ -73,17 +79,14 @@ public void run () {
* @throws QueryException
* @throws KustvaktException
*/
public void loadVCToCache (String filename, String filePath)
throws IOException, QueryException, KustvaktException {
public void loadVCToCache (String filename, String filePath, String json)
throws IOException, QueryException {

InputStream is = NamedVCLoader.class.getResourceAsStream(filePath);
String json = IOUtils.toString(is, "utf-8");
if (json != null) {
cacheVC(filename, json);
vcService.storeQuery("system", filename, ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, json, null, null, null, true,
"system", null, null);
if (json==null || json.isEmpty()) {
InputStream is = NamedVCLoader.class.getResourceAsStream(filePath);
json = IOUtils.toString(is, "utf-8");
}
processVC(filename, json);
}

public void loadVCToCache () throws IOException, QueryException {
Expand All @@ -109,10 +112,60 @@ public void loadVCToCache () throws IOException, QueryException {
filename = strArr[0];
String json = strArr[1];
if (json != null) {
cacheVC(filename, json);
storeVCinDB(filename, json);
processVC(filename, json);
}
}
}

/**
* Stores and caches VC if the given VC does not exist.
* Updates VC in the database and re-caches it, if the given VC exists.
* Updates VC if there is any change in the index.
*
* In this method, it will be checked if
* <ol>
* <li> VC exists in the database</li>
* <li> VC exists in the cache </li>
* <li> KoralQuery of the given VC differs from an existing VC with
* the same id. </li>
* <li> Index has been changed</li>
* </ol>
*
* Koral Query
*
* @param vcId
* @param json
* @throws IOException
* @throws QueryException
*/
private void processVC (String vcId, String json)
throws IOException, QueryException {
boolean updateVC = false;
try {
// if VC exists in the DB
QueryDO existingVC = vcService.searchQueryByName("system", vcId, "system",
QueryType.VIRTUAL_CORPUS);

String koralQuery = existingVC.getKoralQuery();
// if existing VC is different from input
if (json.hashCode() != koralQuery.hashCode()) {
updateVC = true;
}

// updateVCinDB
storeVCinDB(vcId, json, existingVC);
}
catch (KustvaktException e) {
// VC doesn't exist in the DB
if (e.getStatusCode() == StatusCodes.NO_RESOURCE_FOUND) {
storeVCinDB(vcId, json, null);
}
else {
throw new RuntimeException(e);
}
}

cacheVC(vcId, json, updateVC);
}

private String[] readFile (File file, String filename) throws IOException {
Expand Down Expand Up @@ -152,12 +205,18 @@ else if (filename.endsWith(".jsonld.gz")) {
* @param koralQuery
* @throws IOException
* @throws QueryException
* @throws KustvaktException
*/
private void cacheVC (String vcId, String koralQuery)
private void cacheVC (String vcId, String koralQuery, boolean updateVC)
throws IOException, QueryException {
config.setVcInCaching(vcId);
if (VirtualCorpusCache.contains(vcId)) {
if (updateVC) {
jlog.info("Updating {} in cache ", vcId);
VirtualCorpusCache.delete(vcId);
}
else if (VirtualCorpusCache.contains(vcId)) {
jlog.info("Checking {} in cache ", vcId);

}
else {
jlog.info("Storing {} in cache ", vcId);
Expand All @@ -172,32 +231,21 @@ private void cacheVC (String vcId, String koralQuery)
}

/**
* Stores the VC if it doesn't exist in the database.
* Stores new VC or updates existing VC
*
* @param vcId
* @param koralQuery
*/
private void storeVCinDB (String vcId, String koralQuery) {
private void storeVCinDB (String vcId, String koralQuery, QueryDO existingVC) {
try {
vcService.searchQueryByName("system", vcId, "system",
QueryType.VIRTUAL_CORPUS);
jlog.info("Storing {} in database ", vcId);

vcService.storeQuery(existingVC, "system", vcId, ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, koralQuery, null, null, null,
true, "system", null, null);
}
catch (KustvaktException e) {
if (e.getStatusCode() == StatusCodes.NO_RESOURCE_FOUND) {
try {
jlog.info("Storing {} in database ", vcId);
vcService.storeQuery("system", vcId, ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, koralQuery, null, null,
null, true, "system", null, null);
}
catch (KustvaktException e1) {
throw new RuntimeException(e1);
}
}
else {
throw new RuntimeException(e);
}
throw new RuntimeException(e);
}

}
}
23 changes: 20 additions & 3 deletions src/main/java/de/ids_mannheim/korap/service/QueryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,16 @@ public void storeQuery (String username, String queryName,
String definition, String description, String status,
boolean isCached, String queryCreator, String query,
String queryLanguage) throws KustvaktException {
storeQuery(null, username, queryName, type, queryType, koralQuery,
definition, description, status, isCached, queryCreator, query,
queryLanguage);
}

public void storeQuery (QueryDO existingQuery, String username, String queryName,
ResourceType type, QueryType queryType, String koralQuery,
String definition, String description, String status,
boolean isCached, String queryCreator, String query,
String queryLanguage) throws KustvaktException {
ParameterChecker.checkNameValue(queryName, "queryName");
ParameterChecker.checkObjectValue(type, "type");

Expand Down Expand Up @@ -348,9 +358,16 @@ else if (!username.equals("system")) {

int queryId = 0;
try {
queryId = queryDao.createQuery(queryName, type, queryType,
requiredAccess, koralQuery, definition, description, status,
isCached, queryCreator, query, queryLanguage);
if (existingQuery==null) {
queryId = queryDao.createQuery(queryName, type, queryType,
requiredAccess, koralQuery, definition, description,
status, isCached, queryCreator, query, queryLanguage);
}
else {
queryDao.editQuery(existingQuery, queryName, type,
requiredAccess, koralQuery, definition, description,
status, isCached, query, queryLanguage);
}

}
catch (Exception e) {
Expand Down
34 changes: 32 additions & 2 deletions src/test/java/de/ids_mannheim/korap/cache/NamedVCLoaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,40 @@ public void testNamedVCLoader ()
assertTrue(VirtualCorpusCache.contains(vcId));
Map<String, DocBits> cachedData = VirtualCorpusCache.retrieve(vcId);
assertTrue(cachedData.size() > 0);
VirtualCorpusCache.delete(vcId);
assertFalse(VirtualCorpusCache.contains(vcId));
//VirtualCorpusCache.delete(vcId);
//assertFalse(VirtualCorpusCache.contains(vcId));
QueryDO vc = dao.retrieveQueryByName(vcId, "system");
assertNotNull(vc);

String koralQuery = vc.getKoralQuery();
testUpdateVC(vcId,koralQuery);
}

private void testUpdateVC (String vcId, String koralQuery)
throws IOException, QueryException, KustvaktException {
String json = """
{"collection": {
"@type": "koral:doc",
"key": "textSigle",
"match": "match:eq",
"type" : "type:string",
"value": [
"GOE/AGF/00000"
]
}}""";

vcLoader.loadVCToCache(vcId, "", json);

Map<String, DocBits> cachedData = VirtualCorpusCache.retrieve(vcId);
assertTrue(cachedData.size() > 0);

QueryDO vc = dao.retrieveQueryByName(vcId, "system");
String updatedKoralQuery = vc.getKoralQuery();

assertTrue (koralQuery.hashCode() != updatedKoralQuery.hashCode());

VirtualCorpusCache.delete(vcId);
assertFalse(VirtualCorpusCache.contains(vcId));
dao.deleteQuery(vc);
vc = dao.retrieveQueryByName(vcId, "system");
assertNull(vc);
Expand Down

0 comments on commit 4478956

Please sign in to comment.