Skip to content

Commit 70c2d71

Browse files
taban03balhar-jakub
taban03
and
balhar-jakub
authored
Add reject eviction strategy to VSAM (#1112)
* Add mechanism for reject eviction strategy Signed-off-by: at670475 <andrea.tabone@broadcom.com> * Add method to retrieve all the records unbound from service ID Signed-off-by: at670475 <andrea.tabone@broadcom.com> * Temporary change for testing * Add mapping for maxsize in the vsamconfig * Add config to retrieve logs * Log error in proper way * add debug log * attempt * Change allowed Records Num for testing * put back the right allowed records value * temporary change amount of record * set amount of record back * Remove temporary changes * Fix rebasing * Refactoring * Add configuration for the general strategy Signed-off-by: Jakub Balhar <jakub.balhar@broadcom.com> * Remove temporary changes for debug purpose * Fix endline * Add space Signed-off-by: Jakub Balhar <jakub.balhar@broadcom.com> * Change line endings Signed-off-by: Jakub Balhar <jakub.balhar@broadcom.com> Co-authored-by: Jakub Balhar <jakub@balhar.net> Co-authored-by: Jakub Balhar <jakub.balhar@broadcom.com>
1 parent c301456 commit 70c2d71

File tree

8 files changed

+200
-161
lines changed

8 files changed

+200
-161
lines changed

‎caching-service-package/src/main/resources/bin/start.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ _BPX_JOBNAME=${ZOWE_PREFIX}${CACHING_CODE} java -Xms16m -Xmx512m -Xquickstart \
6464
-Dapiml.service.ipAddress=${ZOWE_IP_ADDRESS} \
6565
-Dapiml.service.customMetadata.apiml.gatewayPort=${GATEWAY_PORT} \
6666
-Dapiml.security.ssl.verifySslCertificatesOfServices=${VERIFY_CERTIFICATES} \
67+
-Dcaching.storage.evictionStrategy=${CACHING_EVICTION_STRATEGY} \
68+
-Dcaching.storage.size=${CACHING_STORAGE_SIZE} \
6769
-Denvironment.preferIpAddress=${APIML_PREFER_IP_ADDRESS} \
6870
-Dserver.address=0.0.0.0 \
6971
-Dserver.ssl.enabled=true \

‎caching-service/src/main/java/org/zowe/apiml/caching/config/GeneralConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@
2020
public class GeneralConfig {
2121
@Value("${caching.storage.evictionStrategy:reject}")
2222
private String evictionStrategy;
23+
@Value("${caching.storage.size:100}")
24+
private int maxDataSize;
2325
}

‎caching-service/src/main/java/org/zowe/apiml/caching/service/inmemory/InMemoryStorage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,6 @@ private boolean aboveThreshold() {
116116

117117
log.info("Current Size {}.", currentSize);
118118

119-
return currentSize >= config.getMaxDataSize();
119+
return currentSize >= config.getGeneralConfig().getMaxDataSize();
120120
}
121121
}

‎caching-service/src/main/java/org/zowe/apiml/caching/service/inmemory/config/InMemoryConfig.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import lombok.Data;
1313
import lombok.RequiredArgsConstructor;
1414
import lombok.ToString;
15-
import org.springframework.beans.factory.annotation.Value;
1615
import org.springframework.context.annotation.Configuration;
1716
import org.zowe.apiml.caching.config.GeneralConfig;
1817

@@ -23,7 +22,4 @@
2322
public class InMemoryConfig {
2423
private final GeneralConfig generalConfig;
2524

26-
@Value("${caching.storage.inmemory.size:100}")
27-
private int maxDataSize;
28-
2925
}

‎caching-service/src/main/java/org/zowe/apiml/caching/service/vsam/VsamFile.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,31 @@ public List<VsamRecord> readForService(String serviceId) {
304304
return returned;
305305
}
306306

307+
public Integer countAllRecords() {
308+
int recordsCounter = 0;
309+
310+
try {
311+
byte[] recBuf = new byte[vsamConfig.getRecordLength()];
312+
313+
int overflowProtection = 10000;
314+
while (zfile.read(recBuf) != -1) {
315+
316+
log.trace("RecBuf: {}", recBuf);
317+
318+
recordsCounter += 1;
319+
320+
overflowProtection--;
321+
if (overflowProtection <= 0) {
322+
log.error("Maximum number of records retrieved, stopping the retrieval");
323+
break;
324+
}
325+
}
326+
} catch (ZFileException e) {
327+
log.error(e.toString());
328+
}
329+
return recordsCounter;
330+
}
331+
307332
@SuppressWarnings({"squid:S1130", "squid:S1192"})
308333
private ZFile openZfile() throws ZFileException {
309334
if (!REGEX_CORRECT_FILENAME.matcher(vsamConfig.getFileName()).find()) {

‎caching-service/src/main/java/org/zowe/apiml/caching/service/vsam/VsamStorage.java‎

Lines changed: 167 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,151 +1,167 @@
1-
/*
2-
* This program and the accompanying materials are made available under the terms of the
3-
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4-
* https://www.eclipse.org/legal/epl-v20.html
5-
*
6-
* SPDX-License-Identifier: EPL-2.0
7-
*
8-
* Copyright Contributors to the Zowe Project.
9-
*/
10-
package org.zowe.apiml.caching.service.vsam;
11-
12-
import lombok.extern.slf4j.Slf4j;
13-
import org.springframework.retry.annotation.Retryable;
14-
import org.zowe.apiml.caching.model.KeyValue;
15-
import org.zowe.apiml.caching.service.*;
16-
import org.zowe.apiml.caching.service.vsam.config.VsamConfig;
17-
import org.zowe.apiml.util.ObjectUtil;
18-
19-
import java.util.*;
20-
21-
/**
22-
* Class handles requests from controller and orchestrates operations on the low level VSAM File class
23-
*/
24-
25-
@Slf4j
26-
public class VsamStorage implements Storage {
27-
28-
private final VsamConfig vsamConfig;
29-
30-
public VsamStorage(VsamConfig vsamConfig, VsamInitializer vsamInitializer) {
31-
32-
log.info("Using VSAM storage for the cached data");
33-
34-
ObjectUtil.requireNotNull(vsamConfig.getFileName(), "Vsam filename cannot be null"); //TODO bean validation
35-
ObjectUtil.requireNotEmpty(vsamConfig.getFileName(), "Vsam filename cannot be empty");
36-
this.vsamConfig = vsamConfig;
37-
log.info("Using Vsam configuration: {}", vsamConfig);
38-
vsamInitializer.storageWarmup(vsamConfig);
39-
}
40-
41-
42-
@Override
43-
@Retryable(value = {IllegalStateException.class, UnsupportedOperationException.class})
44-
public KeyValue create(String serviceId, KeyValue toCreate) {
45-
log.info("Writing record: {}|{}|{}", serviceId, toCreate.getKey(), toCreate.getValue());
46-
KeyValue result = null;
47-
48-
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
49-
50-
VsamRecord record = new VsamRecord(vsamConfig, serviceId, toCreate);
51-
52-
Optional<VsamRecord> returned = file.create(record);
53-
if (returned.isPresent()) {
54-
result = returned.get().getKeyValue();
55-
}
56-
57-
}
58-
59-
if (result == null) {
60-
throw new StorageException(Messages.DUPLICATE_KEY.getKey(), Messages.DUPLICATE_KEY.getStatus(), toCreate.getKey(), serviceId);
61-
}
62-
63-
return result;
64-
}
65-
66-
@Override
67-
public KeyValue read(String serviceId, String key) {
68-
log.info("Reading Record: {}|{}|{}", serviceId, key, "-");
69-
KeyValue result = null;
70-
71-
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.READ)) {
72-
73-
VsamRecord record = new VsamRecord(vsamConfig, serviceId, new KeyValue(key, "", serviceId));
74-
75-
Optional<VsamRecord> returned = file.read(record);
76-
if (returned.isPresent()) {
77-
result = returned.get().getKeyValue();
78-
}
79-
}
80-
81-
if (result == null) {
82-
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), key, serviceId);
83-
}
84-
85-
return result;
86-
}
87-
88-
@Override
89-
@Retryable (value = {IllegalStateException.class, UnsupportedOperationException.class})
90-
public KeyValue update(String serviceId, KeyValue toUpdate) {
91-
log.info("Updating Record: {}|{}|{}", serviceId, toUpdate.getKey(), toUpdate.getValue());
92-
KeyValue result = null;
93-
94-
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
95-
96-
VsamRecord record = new VsamRecord(vsamConfig, serviceId, toUpdate);
97-
98-
Optional<VsamRecord> returned = file.update(record);
99-
if (returned.isPresent()) {
100-
result = returned.get().getKeyValue();
101-
}
102-
}
103-
104-
if (result == null) {
105-
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), toUpdate.getKey(), serviceId);
106-
}
107-
108-
return result;
109-
}
110-
111-
@Override
112-
@Retryable(value = {IllegalStateException.class, UnsupportedOperationException.class})
113-
public KeyValue delete(String serviceId, String toDelete) {
114-
115-
log.info("Deleting Record: {}|{}|{}", serviceId, toDelete, "-");
116-
KeyValue result = null;
117-
118-
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
119-
120-
VsamRecord record = new VsamRecord(vsamConfig, serviceId, new KeyValue(toDelete, "", serviceId));
121-
122-
Optional<VsamRecord> returned = file.delete(record);
123-
if (returned.isPresent()) {
124-
result = returned.get().getKeyValue();
125-
}
126-
}
127-
128-
if (result == null) {
129-
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), toDelete, serviceId);
130-
}
131-
132-
return result;
133-
}
134-
135-
@Override
136-
public Map<String, KeyValue> readForService(String serviceId) {
137-
138-
log.info("Reading All Records: {}|{}|{}", serviceId, "-", "-");
139-
Map<String, KeyValue> result = new HashMap<>();
140-
List<VsamRecord> returned;
141-
142-
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.READ)) {
143-
returned = file.readForService(serviceId);
144-
}
145-
146-
returned.forEach(vsamRecord -> result.put(vsamRecord.getKeyValue().getKey(), vsamRecord.getKeyValue()));
147-
148-
return result;
149-
}
150-
151-
}
1+
/*
2+
* This program and the accompanying materials are made available under the terms of the
3+
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4+
* https://www.eclipse.org/legal/epl-v20.html
5+
*
6+
* SPDX-License-Identifier: EPL-2.0
7+
*
8+
* Copyright Contributors to the Zowe Project.
9+
*/
10+
package org.zowe.apiml.caching.service.vsam;
11+
12+
import lombok.extern.slf4j.Slf4j;
13+
import org.springframework.retry.annotation.Retryable;
14+
import org.zowe.apiml.caching.model.KeyValue;
15+
import org.zowe.apiml.caching.service.*;
16+
import org.zowe.apiml.caching.service.inmemory.RejectStrategy;
17+
import org.zowe.apiml.caching.service.vsam.config.VsamConfig;
18+
import org.zowe.apiml.util.ObjectUtil;
19+
20+
import java.util.*;
21+
22+
/**
23+
* Class handles requests from controller and orchestrates operations on the low level VSAM File class
24+
*/
25+
26+
@Slf4j
27+
public class VsamStorage implements Storage {
28+
29+
private final VsamConfig vsamConfig;
30+
private EvictionStrategy strategy = new DefaultEvictionStrategy();
31+
32+
public VsamStorage(VsamConfig vsamConfig, VsamInitializer vsamInitializer) {
33+
String evictionStrategy = vsamConfig.getGeneralConfig().getEvictionStrategy();
34+
log.info("Using VSAM storage for the cached data");
35+
36+
ObjectUtil.requireNotNull(vsamConfig.getFileName(), "Vsam filename cannot be null"); //TODO bean validation
37+
ObjectUtil.requireNotEmpty(vsamConfig.getFileName(), "Vsam filename cannot be empty");
38+
this.vsamConfig = vsamConfig;
39+
if (evictionStrategy.equals(Strategies.REJECT.getKey())) {
40+
strategy = new RejectStrategy();
41+
} else if (evictionStrategy.equals(Strategies.REMOVE_OLDEST.getKey())) {
42+
// strategy = new RemoveOldestStrategy(storage);
43+
}
44+
log.info("Using Vsam configuration: {}", vsamConfig);
45+
vsamInitializer.storageWarmup(vsamConfig);
46+
}
47+
48+
49+
@Override
50+
@Retryable(value = {IllegalStateException.class, UnsupportedOperationException.class})
51+
public KeyValue create(String serviceId, KeyValue toCreate) {
52+
log.info("Writing record: {}|{}|{}", serviceId, toCreate.getKey(), toCreate.getValue());
53+
KeyValue result = null;
54+
55+
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
56+
57+
VsamRecord record = new VsamRecord(vsamConfig, serviceId, toCreate);
58+
int currentSize = file.countAllRecords();
59+
log.info("Current Size {}.", currentSize);
60+
61+
if (aboveThreshold(currentSize)) {
62+
log.info("Rejecting record");
63+
strategy.evict(toCreate.getKey());
64+
}
65+
Optional<VsamRecord> returned = file.create(record);
66+
if (returned.isPresent()) {
67+
result = returned.get().getKeyValue();
68+
}
69+
}
70+
71+
if (result == null) {
72+
throw new StorageException(Messages.DUPLICATE_KEY.getKey(), Messages.DUPLICATE_KEY.getStatus(), toCreate.getKey(), serviceId);
73+
}
74+
75+
return result;
76+
}
77+
78+
private boolean aboveThreshold(int currentSize) {
79+
return currentSize >= vsamConfig.getGeneralConfig().getMaxDataSize();
80+
}
81+
82+
@Override
83+
public KeyValue read(String serviceId, String key) {
84+
log.info("Reading Record: {}|{}|{}", serviceId, key, "-");
85+
KeyValue result = null;
86+
87+
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.READ)) {
88+
89+
VsamRecord record = new VsamRecord(vsamConfig, serviceId, new KeyValue(key, "", serviceId));
90+
91+
Optional<VsamRecord> returned = file.read(record);
92+
if (returned.isPresent()) {
93+
result = returned.get().getKeyValue();
94+
}
95+
}
96+
97+
if (result == null) {
98+
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), key, serviceId);
99+
}
100+
101+
return result;
102+
}
103+
104+
@Override
105+
@Retryable(value = {IllegalStateException.class, UnsupportedOperationException.class})
106+
public KeyValue update(String serviceId, KeyValue toUpdate) {
107+
log.info("Updating Record: {}|{}|{}", serviceId, toUpdate.getKey(), toUpdate.getValue());
108+
KeyValue result = null;
109+
110+
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
111+
112+
VsamRecord record = new VsamRecord(vsamConfig, serviceId, toUpdate);
113+
114+
Optional<VsamRecord> returned = file.update(record);
115+
if (returned.isPresent()) {
116+
result = returned.get().getKeyValue();
117+
}
118+
}
119+
120+
if (result == null) {
121+
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), toUpdate.getKey(), serviceId);
122+
}
123+
124+
return result;
125+
}
126+
127+
@Override
128+
@Retryable(value = {IllegalStateException.class, UnsupportedOperationException.class})
129+
public KeyValue delete(String serviceId, String toDelete) {
130+
131+
log.info("Deleting Record: {}|{}|{}", serviceId, toDelete, "-");
132+
KeyValue result = null;
133+
134+
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.WRITE)) {
135+
136+
VsamRecord record = new VsamRecord(vsamConfig, serviceId, new KeyValue(toDelete, "", serviceId));
137+
138+
Optional<VsamRecord> returned = file.delete(record);
139+
if (returned.isPresent()) {
140+
result = returned.get().getKeyValue();
141+
}
142+
}
143+
144+
if (result == null) {
145+
throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), toDelete, serviceId);
146+
}
147+
148+
return result;
149+
}
150+
151+
@Override
152+
public Map<String, KeyValue> readForService(String serviceId) {
153+
154+
log.info("Reading All Records: {}|{}|{}", serviceId, "-", "-");
155+
Map<String, KeyValue> result = new HashMap<>();
156+
List<VsamRecord> returned;
157+
158+
try (VsamFile file = new VsamFile(vsamConfig, VsamConfig.VsamOptions.READ)) {
159+
returned = file.readForService(serviceId);
160+
}
161+
162+
returned.forEach(vsamRecord -> result.put(vsamRecord.getKeyValue().getKey(), vsamRecord.getKeyValue()));
163+
164+
return result;
165+
}
166+
167+
}

0 commit comments

Comments
 (0)