Skip to content

Commit fbe60ca

Browse files
AlexisDrogoulAlexisDrogoul
committed
Should fix #3786 and some additional bugs found on CSV files
Should fix #3786 and some additional bugs found on CSV files, especially the ones with exotic text qualifiers / delimiters. Adds two preferences for setting the default delimiter and default text qualifier.
1 parent 7ef8274 commit fbe60ca

File tree

13 files changed

+657
-2009
lines changed

13 files changed

+657
-2009
lines changed

‎msi.gama.core/src/msi/gama/common/preferences/GamaPreferences.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import msi.gama.util.GamaMapFactory;
4141
import msi.gama.util.file.GenericFile;
4242
import msi.gama.util.file.IGamaFile;
43+
import msi.gama.util.file.csv.AbstractCSVManipulator;
4344
import msi.gaml.compilation.GAML;
4445
import msi.gaml.compilation.kernel.GamaMetaModel;
4546
import msi.gaml.operators.Cast;
@@ -59,6 +60,10 @@
5960
* The Class GamaPreferences.
6061
*/
6162

63+
/**
64+
* The Class GamaPreferences.
65+
*/
66+
6267
/**
6368
* The Class GamaPreferences.
6469
*/
@@ -352,10 +357,12 @@ public static class Runtime {
352357
// public static final Pref<Double> CORE_DELAY_STEP = create("pref_experiment_default_step",
353358
/** The Constant CORE_SYNC. */
354359
// "Default step for the delay slider (in sec.)", 0.001, IType.FLOAT, true).in(NAME, EXECUTION).disabled();
355-
360+
356361
public static final Pref<Boolean> CORE_SLIDER_TYPE = create("pref_experiment_type_slider",
357-
"Set the step duration slider incrementation to linear. If false set to logarithmic", true, IType.BOOL, true).in(NAME, EXECUTION);
358-
362+
"Set the step duration slider incrementation to linear. If false set to logarithmic", true, IType.BOOL,
363+
true).in(NAME, EXECUTION);
364+
365+
/** The Constant CORE_SYNC. */
359366
public static final Pref<Boolean> CORE_SYNC =
360367
create("pref_display_synchronized", "Synchronize outputs with the simulation", false, IType.BOOL, true)
361368
.in(NAME, EXECUTION);
@@ -815,6 +822,16 @@ public static class External {
815822
if (codes.isEmpty()) return false;
816823
return true;
817824
});
825+
826+
/** The Constant CSV_STRING_QUALIFIER. */
827+
public static final Pref<String> CSV_STRING_QUALIFIER = GamaPreferences
828+
.create("pref_csv_string_qualifier", "Default separator for strings", String.valueOf(AbstractCSVManipulator.Letters.QUOTE), IType.STRING, true)
829+
.in(NAME, "CSV Files");
830+
831+
/** The Constant CSV_SEPARATOR. */
832+
public static final Pref<String> CSV_SEPARATOR =
833+
GamaPreferences.create("pref_csv_separator", "Default separator for fields", String.valueOf(AbstractCSVManipulator.Letters.COMMA), IType.STRING, true)
834+
.in(GamaPreferences.External.NAME, "CSV Files");
818835
}
819836

820837
/**

‎msi.gama.core/src/msi/gama/outputs/FileOutput.java

Lines changed: 76 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
/*******************************************************************************************************
22
*
3-
* FileOutput.java, in msi.gama.core, is part of the source code of the
4-
* GAMA modeling and simulation platform (v.1.9.2).
3+
* FileOutput.java, in msi.gama.core, is part of the source code of the GAMA modeling and simulation platform (v.1.9.2).
54
*
65
* (c) 2007-2023 UMI 209 UMMISCO IRD/SU & Partners (IRIT, MIAT, TLU, CTU)
76
*
87
* Visit https://github.com/gama-platform/gama for license information and contacts.
9-
*
8+
*
109
********************************************************************************************************/
1110
package msi.gama.outputs;
1211

@@ -19,7 +18,6 @@
1918
import java.util.Arrays;
2019
import java.util.Calendar;
2120
import java.util.List;
22-
import java.util.stream.Collectors;
2321

2422
import msi.gama.common.interfaces.IKeyword;
2523
import msi.gama.kernel.experiment.IExperimentPlan;
@@ -34,6 +32,7 @@
3432
import msi.gama.runtime.IScope;
3533
import msi.gama.runtime.exceptions.GamaRuntimeException;
3634
import msi.gama.util.IMap;
35+
import msi.gama.util.file.csv.AbstractCSVManipulator;
3736
import msi.gaml.compilation.GAML;
3837
import msi.gaml.descriptions.IDescription;
3938
import msi.gaml.expressions.IExpression;
@@ -46,6 +45,7 @@
4645
* </p>
4746
* A particular output file especially design for the batch experiment output
4847
* </p>
48+
*
4949
* @author drogoul
5050
*/
5151
@symbol (
@@ -125,55 +125,55 @@ public FileOutput(/* final ISymbol context, */final IDescription desc) {
125125

126126
/** The file. */
127127
File file = null;
128-
128+
129129
/** The file name. */
130130
String fileName = "";
131-
131+
132132
/** The rewrite. */
133133
boolean rewrite = false;
134-
134+
135135
/** The header. */
136136
String header = "";
137-
137+
138138
/** The footer. */
139139
String footer = "";
140-
140+
141141
/** The last value. */
142142
Object lastValue = null;
143-
143+
144144
/** The last values. */
145145
List<Object> lastValues = null;
146-
146+
147147
/** The logged batch param. */
148148
List<String> loggedBatchParam = null;
149-
149+
150150
/** The solution. */
151151
ParametersSet solution = null;
152-
152+
153153
/** The expression text. */
154154
private String expressionText = null;
155-
155+
156156
/** The data. */
157157
private IExpression data;
158-
158+
159159
/** The Constant LOG_FOLDER. */
160160
private static final String LOG_FOLDER = "log";
161-
161+
162162
/** The Constant XMLHeader. */
163163
private static final String XMLHeader = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?>";
164-
164+
165165
/** The Constant XML. */
166166
private static final int XML = 1;
167-
167+
168168
/** The Constant CSV. */
169169
private static final int CSV = 2;
170-
170+
171171
/** The Constant TEXT. */
172172
private static final int TEXT = 0;
173-
173+
174174
/** The Constant extensions. */
175175
private static final List<String> extensions = Arrays.asList("txt", "xml", "csv");
176-
176+
177177
/** The type. */
178178
private int type;
179179

@@ -198,7 +198,8 @@ private void createExpression() {
198198
/**
199199
* Creates the header.
200200
*
201-
* @throws GamaRuntimeException the gama runtime exception
201+
* @throws GamaRuntimeException
202+
* the gama runtime exception
202203
*/
203204
private void createHeader() throws GamaRuntimeException {
204205
final IExpression exp = getFacet(IKeyword.HEADER);
@@ -212,7 +213,8 @@ private void createHeader() throws GamaRuntimeException {
212213
/**
213214
* Creates the footer.
214215
*
215-
* @throws GamaRuntimeException the gama runtime exception
216+
* @throws GamaRuntimeException
217+
* the gama runtime exception
216218
*/
217219
private void createFooter() throws GamaRuntimeException {
218220
final IExpression exp = getFacet(IKeyword.FOOTER);
@@ -226,7 +228,8 @@ private void createFooter() throws GamaRuntimeException {
226228
/**
227229
* Creates the rewrite.
228230
*
229-
* @throws GamaRuntimeException the gama runtime exception
231+
* @throws GamaRuntimeException
232+
* the gama runtime exception
230233
*/
231234
private void createRewrite() throws GamaRuntimeException {
232235
final IExpression exp = getFacet(IKeyword.REWRITE);
@@ -307,11 +310,16 @@ public boolean init(final IScope scope) throws GamaRuntimeException {
307310
/**
308311
* Instantiates a new file output.
309312
*
310-
* @param name the name
311-
* @param expr the expr
312-
* @param columns the columns
313-
* @param exp the exp
314-
* @throws GamaRuntimeException the gama runtime exception
313+
* @param name
314+
* the name
315+
* @param expr
316+
* the expr
317+
* @param columns
318+
* the columns
319+
* @param exp
320+
* the exp
321+
* @throws GamaRuntimeException
322+
* the gama runtime exception
315323
*/
316324
public FileOutput(final String name, final String expr, final List<String> columns, final IExperimentPlan exp)
317325
throws GamaRuntimeException {
@@ -380,7 +388,8 @@ private void createFileName(final IScope scope) throws GamaRuntimeException {
380388
/**
381389
* Refresh expression.
382390
*
383-
* @throws GamaRuntimeException the gama runtime exception
391+
* @throws GamaRuntimeException
392+
* the gama runtime exception
384393
*/
385394
public void refreshExpression() throws GamaRuntimeException {
386395
// in case the file writer persists over different simulations (like in
@@ -407,11 +416,12 @@ public boolean step(final IScope scope) {
407416
public void update() throws GamaRuntimeException {
408417
writeToFile(getScope().getClock().getCycle());
409418
}
410-
419+
411420
/**
412421
* Do write report and close.
413422
*
414-
* @param report the report
423+
* @param report
424+
* the report
415425
*/
416426
public void doWriteReportAndClose(final String report) {
417427
switch (type) {
@@ -429,19 +439,21 @@ public void doWriteReportAndClose(final String report) {
429439
break;
430440
}
431441
}
432-
442+
433443
/**
434444
* Main method to write down a set of given values for a single a point in the parameter space
445+
*
435446
* @param sol
436447
* @param outputs
437448
* @throws GamaRuntimeException
438449
*/
439-
public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Object> outputs) throws GamaRuntimeException {
450+
public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String, Object> outputs)
451+
throws GamaRuntimeException {
440452
setSolution(sol);
441453
if (outputs == null || outputs.isEmpty()) {
442-
if (!getScope().step(this).passed()) { return; }
454+
if (!getScope().step(this).passed()) return;
443455
} else {
444-
this.lastValues = outputs.values().stream().toList(); //setLastValue(fitness);
456+
this.lastValues = outputs.values().stream().toList(); // setLastValue(fitness);
445457
}
446458
// compute(getOwnScope(), 0l);
447459
switch (type) {
@@ -462,11 +474,15 @@ public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Ob
462474
case CSV:
463475
if (solution == null) return;
464476
final StringBuilder s = new StringBuilder(loggedBatchParam.size() * 8);
465-
for (final String var : loggedBatchParam) { s.append(solution.get(var)).append(','); }
477+
for (final String var : loggedBatchParam) {
478+
s.append(solution.get(var)).append(AbstractCSVManipulator.getDefaultDelimiter());
479+
}
466480
if (lastValue != null) {
467481
s.append(lastValue);
468482
} else if (lastValues != null && !lastValues.isEmpty()) {
469-
for (Object val : lastValues) {s.append(',').append(val); }
483+
for (Object val : lastValues) {
484+
s.append(AbstractCSVManipulator.getDefaultDelimiter()).append(val);
485+
}
470486
} else {
471487
s.setLength(s.length() - 1);
472488
}
@@ -485,7 +501,8 @@ public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Ob
485501
/**
486502
* Write to file.
487503
*
488-
* @param cycle the cycle
504+
* @param cycle
505+
* the cycle
489506
*/
490507
void writeToFile(final long cycle) {
491508
switch (type) {
@@ -512,7 +529,8 @@ void writeToFile(final long cycle) {
512529
/**
513530
* Sets the rewrite.
514531
*
515-
* @param rewrite the new rewrite
532+
* @param rewrite
533+
* the new rewrite
516534
*/
517535
private void setRewrite(final boolean rewrite) { this.rewrite = rewrite; }
518536

@@ -532,7 +550,8 @@ private String getHeader() {
532550
/**
533551
* Sets the header.
534552
*
535-
* @param header the new header
553+
* @param header
554+
* the new header
536555
*/
537556
private void setHeader(final String header) { this.header = header; }
538557

@@ -552,7 +571,8 @@ private String getFooter() {
552571
/**
553572
* Sets the footer.
554573
*
555-
* @param footer the new footer
574+
* @param footer
575+
* the new footer
556576
*/
557577
private void setFooter(final String footer) { this.footer = footer; }
558578

@@ -566,14 +586,16 @@ private String getFooter() {
566586
/**
567587
* Sets the writer.
568588
*
569-
* @param writer the new writer
589+
* @param writer
590+
* the new writer
570591
*/
571592
private void setWriter(final PrintWriter writer) { this.writer = writer; }
572593

573594
/**
574595
* Sets the last value.
575596
*
576-
* @param lastValue the new last value
597+
* @param lastValue
598+
* the new last value
577599
*/
578600
public void setLastValue(final Object lastValue) { this.lastValue = lastValue; }
579601

@@ -587,9 +609,12 @@ private String getFooter() {
587609
/**
588610
* Sets the logged batch param.
589611
*
590-
* @param loggedBatchParam the new logged batch param
612+
* @param loggedBatchParam
613+
* the new logged batch param
591614
*/
592-
public void setLoggedBatchParam(final List<String> loggedBatchParam) { this.loggedBatchParam = loggedBatchParam; }
615+
public void setLoggedBatchParam(final List<String> loggedBatchParam) {
616+
this.loggedBatchParam = loggedBatchParam;
617+
}
593618

594619
/**
595620
* Gets the solution.
@@ -601,7 +626,8 @@ private String getFooter() {
601626
/**
602627
* Sets the solution.
603628
*
604-
* @param solution the new solution
629+
* @param solution
630+
* the new solution
605631
*/
606632
public void setSolution(final ParametersSet solution) { this.solution = solution; }
607633

@@ -622,7 +648,9 @@ public void writeHeaderAndClose() {
622648
break;
623649
case CSV:
624650
final StringBuilder s = new StringBuilder(loggedBatchParam.size() * 8);
625-
for (final String var : loggedBatchParam) { s.append(var).append(','); }
651+
for (final String var : loggedBatchParam) {
652+
s.append(var).append(AbstractCSVManipulator.getDefaultDelimiter());
653+
}
626654
if (getFacet(IKeyword.DATA) != null) {
627655
s.append(getLiteral(IKeyword.DATA));
628656
} else {

‎msi.gama.core/src/msi/gama/outputs/MonitorOutput.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,8 @@ private void setColor(final IExpression facet) {
148148
/**
149149
* Sets the color.
150150
*
151-
* @param gamaColor the new color
151+
* @param gamaColor
152+
* the new color
152153
*/
153154
public void setColor(final GamaColor gamaColor) {
154155
color = gamaColor;
@@ -272,7 +273,7 @@ public void saveHistory() {
272273
monitorFolder + "/" + "monitor_" + getName() + "_cycle_" + getScope().getClock().getCycle() + ".csv";
273274
file = FileUtils.constructAbsoluteFilePath(getScope(), file, false);
274275
try (final BufferedWriter bw = new BufferedWriter(new FileWriter(file));
275-
final CsvWriter w = new CsvWriter(bw, CsvWriter.Letters.COMMA)) {
276+
final CsvWriter w = new CsvWriter(bw)) {
276277
for (final Object o : history) {
277278
String[] strings = null;
278279
if (o instanceof Number) {

0 commit comments

Comments
 (0)