Smaller memory footprint

ceskaexpedice · Jul 14, 2021 · 965ef70 · 965ef70
1 parent 261de4a
commit 965ef70
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 28 deletions.
diff --git a/common/src/main/java/cz/incad/kramerius/processes/res/lp.st b/common/src/main/java/cz/incad/kramerius/processes/res/lp.st
@@ -663,7 +663,7 @@
             <standardOs>lrOut</standardOs>
             <!-- err vystup -->
             <errOs>lrErr</errOs>
-            <javaProcessParameters>-Xmx1024m -Xms256m</javaProcessParameters>
+            <javaProcessParameters>-Xmx4096m -Xms256m -Dfile.encoding=UTF-8</javaProcessParameters>
             <templates>
                 <input class="cz.incad.kramerius.statistics.impl.nkp.ParametrizedNKPInputTemplate"></input>
             </templates>

diff --git a/nkp-logs/src/main/java/cz/incad/kramerius/statistics/impl/nkp/NKPLogProcess.java b/nkp-logs/src/main/java/cz/incad/kramerius/statistics/impl/nkp/NKPLogProcess.java
@@ -75,44 +75,60 @@ public static void  process(@ParameterName("dateFrom")String from,
 
 
             WebResource.Builder builder = r.header("auth-token", authHeader).header("token", groupHeader).accept(MediaType.APPLICATION_JSON);
-            String content = builder.get(String.class);
+            InputStream clientResponse = builder.get(InputStream.class);
+
 
             File outputFolder = new File(folder);
             if (!outputFolder.exists())  outputFolder.mkdirs();
             if (outputFolder.exists()) {
 
-                StringBuilder stringBuilder = new StringBuilder();
-
-                BufferedReader bufferedReader = new BufferedReader(new StringReader(content));
-                String line = null;
-                while((line = bufferedReader.readLine()) != null) {
-
-                    JSONObject lineJSONObject = new JSONObject(line);
-                    annonymizationKeys.stream().forEach(key-> {
-                        if (lineJSONObject.has(key)) {
-                            Object o = lineJSONObject.get(key);
-                            if (o instanceof  String) {
-                                String stringToBeHashed = o.toString();
-                                String newVal = null;
-                                if (stringToBeHashed.contains("@")) {
-                                    String[] split = stringToBeHashed.split("@");
-                                    newVal = String.format("%s@%s", hashVal(split[0]), hashVal(split[1]));
-                                } else {
-                                    newVal = hashVal(stringToBeHashed);
+                File outputFile = new File(outputFolder, String.format("statistics-%s-%s.log", institution, StatisticReport.DATE_FORMAT.format(processingDate)));
+                FileOutputStream fos = new FileOutputStream(outputFile);
+                try(OutputStreamWriter fileWriter = new OutputStreamWriter(fos, "UTF-8")) {
+
+                    BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(clientResponse, "UTF-8"));
+                    String line = null;
+                    while ((line = bufferedReader.readLine()) != null) {
+
+                        JSONObject lineJSONObject = new JSONObject(line);
+                        annonymizationKeys.stream().forEach(key -> {
+                            if (lineJSONObject.has(key)) {
+                                Object o = lineJSONObject.get(key);
+                                if (o instanceof String) {
+                                    String stringToBeHashed = o.toString();
+                                    String newVal = null;
+                                    if (stringToBeHashed.contains("@")) {
+                                        String[] split = stringToBeHashed.split("@");
+                                        newVal = String.format("%s@%s", hashVal(split[0]), hashVal(split[1]));
+                                    } else {
+                                        newVal = hashVal(stringToBeHashed);
+                                    }
+                                    lineJSONObject.put(key, newVal);
                                 }
-                                lineJSONObject.put(key, newVal);
                             }
-                        }
-                    });
+                        });
 
-                    stringBuilder.append(lineJSONObject.toString()).append("\n");
-                }
+                        fileWriter.write(lineJSONObject.toString()+"\n");
 
+                        // memory dump
+                        boolean nkpLogMemoryDump = KConfiguration.getInstance().getConfiguration().getBoolean("nkp.logs.memorydump",false);
+                        if (nkpLogMemoryDump) {
+                            Runtime runtime = Runtime.getRuntime();
+                            long freeMemory = runtime.freeMemory();
+                            long totalMemory = runtime.totalMemory();
+
+                            long usedMemory = totalMemory - freeMemory;
+
+                            long maxMemory = runtime.maxMemory();
+                            double ratioA = (double) usedMemory/(double) totalMemory;
+                            double ratioB = (double) usedMemory/(double) maxMemory;
+
+                            LOGGER.info(String.format("Free memory (bytes) %d,Used memory (bytes) %d, Total memory(bytes) %d, Max memory (bytes) %d, ratio (used/totalmemory)  %f,ratio (used/maxmemory)  %f", freeMemory, usedMemory, totalMemory, maxMemory, ratioA,ratioB));
+                        }
+                    }
+                }
 
-                File outputFile = new File(outputFolder, String.format("statistics-%s-%s.log", institution, StatisticReport.DATE_FORMAT.format(processingDate)));
                 LOGGER.info(String.format("Storing log to file %s",outputFile.getAbsolutePath()));
-                IOUtils.saveToFile(stringBuilder.toString(), outputFile);
-                //IOUtils.saveToFile(jsonObject.toString(), outputFile);
             }
             processingDate = nextDate;