Skip to content

Commit

Permalink
Should fix #3786 and some additional bugs found on CSV files
Browse files Browse the repository at this point in the history 
Should fix #3786 and some additional bugs found on CSV files, especially
the ones with exotic text qualifiers / delimiters.
Adds two preferences for setting the default delimiter and default text
qualifier.
  • Loading branch information
@AlexisDrogoul
AlexisDrogoul committed Apr 28, 2023
1 parent 7ef8274 commit fbe60ca
Show file tree
Hide file tree
Showing 13 changed files with 657 additions and 2,009 deletions.
23 changes: 20 additions & 3 deletions msi.gama.core/src/msi/gama/common/preferences/GamaPreferences.java
View file
Expand Up @@ -40,6 +40,7 @@
import msi.gama.util.GamaMapFactory;
import msi.gama.util.file.GenericFile;
import msi.gama.util.file.IGamaFile;
import msi.gama.util.file.csv.AbstractCSVManipulator;
import msi.gaml.compilation.GAML;
import msi.gaml.compilation.kernel.GamaMetaModel;
import msi.gaml.operators.Cast;
Expand All @@ -59,6 +60,10 @@
* The Class GamaPreferences.
*/

/**
* The Class GamaPreferences.
*/

/**
* The Class GamaPreferences.
*/
Expand Down Expand Up @@ -352,10 +357,12 @@ public static class Runtime {
// public static final Pref<Double> CORE_DELAY_STEP = create("pref_experiment_default_step",
/** The Constant CORE_SYNC. */
// "Default step for the delay slider (in sec.)", 0.001, IType.FLOAT, true).in(NAME, EXECUTION).disabled();

public static final Pref<Boolean> CORE_SLIDER_TYPE = create("pref_experiment_type_slider",
"Set the step duration slider incrementation to linear. If false set to logarithmic", true, IType.BOOL, true).in(NAME, EXECUTION);

"Set the step duration slider incrementation to linear. If false set to logarithmic", true, IType.BOOL,
true).in(NAME, EXECUTION);

/** The Constant CORE_SYNC. */
public static final Pref<Boolean> CORE_SYNC =
create("pref_display_synchronized", "Synchronize outputs with the simulation", false, IType.BOOL, true)
.in(NAME, EXECUTION);
Expand Down Expand Up @@ -815,6 +822,16 @@ public static class External {
if (codes.isEmpty()) return false;
return true;
});

/** The Constant CSV_STRING_QUALIFIER. */
public static final Pref<String> CSV_STRING_QUALIFIER = GamaPreferences
.create("pref_csv_string_qualifier", "Default separator for strings", String.valueOf(AbstractCSVManipulator.Letters.QUOTE), IType.STRING, true)
.in(NAME, "CSV Files");

/** The Constant CSV_SEPARATOR. */
public static final Pref<String> CSV_SEPARATOR =
GamaPreferences.create("pref_csv_separator", "Default separator for fields", String.valueOf(AbstractCSVManipulator.Letters.COMMA), IType.STRING, true)
.in(GamaPreferences.External.NAME, "CSV Files");
}

/**
Expand Down
124 changes: 76 additions & 48 deletions msi.gama.core/src/msi/gama/outputs/FileOutput.java
View file
@@ -1,12 +1,11 @@
/*******************************************************************************************************
*
* FileOutput.java, in msi.gama.core, is part of the source code of the
* GAMA modeling and simulation platform (v.1.9.2).
* FileOutput.java, in msi.gama.core, is part of the source code of the GAMA modeling and simulation platform (v.1.9.2).
*
* (c) 2007-2023 UMI 209 UMMISCO IRD/SU & Partners (IRIT, MIAT, TLU, CTU)
*
* Visit https://github.com/gama-platform/gama for license information and contacts.
*
*
********************************************************************************************************/
package msi.gama.outputs;

Expand All @@ -19,7 +18,6 @@
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;
import java.util.stream.Collectors;

import msi.gama.common.interfaces.IKeyword;
import msi.gama.kernel.experiment.IExperimentPlan;
Expand All @@ -34,6 +32,7 @@
import msi.gama.runtime.IScope;
import msi.gama.runtime.exceptions.GamaRuntimeException;
import msi.gama.util.IMap;
import msi.gama.util.file.csv.AbstractCSVManipulator;
import msi.gaml.compilation.GAML;
import msi.gaml.descriptions.IDescription;
import msi.gaml.expressions.IExpression;
Expand All @@ -46,6 +45,7 @@
* </p>
* A particular output file especially design for the batch experiment output
* </p>
*
* @author drogoul
*/
@symbol (
Expand Down Expand Up @@ -125,55 +125,55 @@ public FileOutput(/* final ISymbol context, */final IDescription desc) {

/** The file. */
File file = null;

/** The file name. */
String fileName = "";

/** The rewrite. */
boolean rewrite = false;

/** The header. */
String header = "";

/** The footer. */
String footer = "";

/** The last value. */
Object lastValue = null;

/** The last values. */
List<Object> lastValues = null;

/** The logged batch param. */
List<String> loggedBatchParam = null;

/** The solution. */
ParametersSet solution = null;

/** The expression text. */
private String expressionText = null;

/** The data. */
private IExpression data;

/** The Constant LOG_FOLDER. */
private static final String LOG_FOLDER = "log";

/** The Constant XMLHeader. */
private static final String XMLHeader = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?>";

/** The Constant XML. */
private static final int XML = 1;

/** The Constant CSV. */
private static final int CSV = 2;

/** The Constant TEXT. */
private static final int TEXT = 0;

/** The Constant extensions. */
private static final List<String> extensions = Arrays.asList("txt", "xml", "csv");

/** The type. */
private int type;

Expand All @@ -198,7 +198,8 @@ private void createExpression() {
/**
* Creates the header.
*
* @throws GamaRuntimeException the gama runtime exception
* @throws GamaRuntimeException
* the gama runtime exception
*/
private void createHeader() throws GamaRuntimeException {
final IExpression exp = getFacet(IKeyword.HEADER);
Expand All @@ -212,7 +213,8 @@ private void createHeader() throws GamaRuntimeException {
/**
* Creates the footer.
*
* @throws GamaRuntimeException the gama runtime exception
* @throws GamaRuntimeException
* the gama runtime exception
*/
private void createFooter() throws GamaRuntimeException {
final IExpression exp = getFacet(IKeyword.FOOTER);
Expand All @@ -226,7 +228,8 @@ private void createFooter() throws GamaRuntimeException {
/**
* Creates the rewrite.
*
* @throws GamaRuntimeException the gama runtime exception
* @throws GamaRuntimeException
* the gama runtime exception
*/
private void createRewrite() throws GamaRuntimeException {
final IExpression exp = getFacet(IKeyword.REWRITE);
Expand Down Expand Up @@ -307,11 +310,16 @@ public boolean init(final IScope scope) throws GamaRuntimeException {
/**
* Instantiates a new file output.
*
* @param name the name
* @param expr the expr
* @param columns the columns
* @param exp the exp
* @throws GamaRuntimeException the gama runtime exception
* @param name
* the name
* @param expr
* the expr
* @param columns
* the columns
* @param exp
* the exp
* @throws GamaRuntimeException
* the gama runtime exception
*/
public FileOutput(final String name, final String expr, final List<String> columns, final IExperimentPlan exp)
throws GamaRuntimeException {
Expand Down Expand Up @@ -380,7 +388,8 @@ private void createFileName(final IScope scope) throws GamaRuntimeException {
/**
* Refresh expression.
*
* @throws GamaRuntimeException the gama runtime exception
* @throws GamaRuntimeException
* the gama runtime exception
*/
public void refreshExpression() throws GamaRuntimeException {
// in case the file writer persists over different simulations (like in
Expand All @@ -407,11 +416,12 @@ public boolean step(final IScope scope) {
public void update() throws GamaRuntimeException {
writeToFile(getScope().getClock().getCycle());
}

/**
* Do write report and close.
*
* @param report the report
* @param report
* the report
*/
public void doWriteReportAndClose(final String report) {
switch (type) {
Expand All @@ -429,19 +439,21 @@ public void doWriteReportAndClose(final String report) {
break;
}
}

/**
* Main method to write down a set of given values for a single a point in the parameter space
*
* @param sol
* @param outputs
* @throws GamaRuntimeException
*/
public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Object> outputs) throws GamaRuntimeException {
public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String, Object> outputs)
throws GamaRuntimeException {
setSolution(sol);
if (outputs == null || outputs.isEmpty()) {
if (!getScope().step(this).passed()) { return; }
if (!getScope().step(this).passed()) return;
} else {
this.lastValues = outputs.values().stream().toList(); //setLastValue(fitness);
this.lastValues = outputs.values().stream().toList(); // setLastValue(fitness);
}
// compute(getOwnScope(), 0l);
switch (type) {
Expand All @@ -462,11 +474,15 @@ public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Ob
case CSV:
if (solution == null) return;
final StringBuilder s = new StringBuilder(loggedBatchParam.size() * 8);
for (final String var : loggedBatchParam) { s.append(solution.get(var)).append(','); }
for (final String var : loggedBatchParam) {
s.append(solution.get(var)).append(AbstractCSVManipulator.getDefaultDelimiter());
}
if (lastValue != null) {
s.append(lastValue);
} else if (lastValues != null && !lastValues.isEmpty()) {
for (Object val : lastValues) {s.append(',').append(val); }
for (Object val : lastValues) {
s.append(AbstractCSVManipulator.getDefaultDelimiter()).append(val);
}
} else {
s.setLength(s.length() - 1);
}
Expand All @@ -485,7 +501,8 @@ public void doRefreshWriteAndClose(final ParametersSet sol, final IMap<String,Ob
/**
* Write to file.
*
* @param cycle the cycle
* @param cycle
* the cycle
*/
void writeToFile(final long cycle) {
switch (type) {
Expand All @@ -512,7 +529,8 @@ void writeToFile(final long cycle) {
/**
* Sets the rewrite.
*
* @param rewrite the new rewrite
* @param rewrite
* the new rewrite
*/
private void setRewrite(final boolean rewrite) { this.rewrite = rewrite; }

Expand All @@ -532,7 +550,8 @@ private String getHeader() {
/**
* Sets the header.
*
* @param header the new header
* @param header
* the new header
*/
private void setHeader(final String header) { this.header = header; }

Expand All @@ -552,7 +571,8 @@ private String getFooter() {
/**
* Sets the footer.
*
* @param footer the new footer
* @param footer
* the new footer
*/
private void setFooter(final String footer) { this.footer = footer; }

Expand All @@ -566,14 +586,16 @@ private String getFooter() {
/**
* Sets the writer.
*
* @param writer the new writer
* @param writer
* the new writer
*/
private void setWriter(final PrintWriter writer) { this.writer = writer; }

/**
* Sets the last value.
*
* @param lastValue the new last value
* @param lastValue
* the new last value
*/
public void setLastValue(final Object lastValue) { this.lastValue = lastValue; }

Expand All @@ -587,9 +609,12 @@ private String getFooter() {
/**
* Sets the logged batch param.
*
* @param loggedBatchParam the new logged batch param
* @param loggedBatchParam
* the new logged batch param
*/
public void setLoggedBatchParam(final List<String> loggedBatchParam) { this.loggedBatchParam = loggedBatchParam; }
public void setLoggedBatchParam(final List<String> loggedBatchParam) {
this.loggedBatchParam = loggedBatchParam;
}

/**
* Gets the solution.
Expand All @@ -601,7 +626,8 @@ private String getFooter() {
/**
* Sets the solution.
*
* @param solution the new solution
* @param solution
* the new solution
*/
public void setSolution(final ParametersSet solution) { this.solution = solution; }

Expand All @@ -622,7 +648,9 @@ public void writeHeaderAndClose() {
break;
case CSV:
final StringBuilder s = new StringBuilder(loggedBatchParam.size() * 8);
for (final String var : loggedBatchParam) { s.append(var).append(','); }
for (final String var : loggedBatchParam) {
s.append(var).append(AbstractCSVManipulator.getDefaultDelimiter());
}
if (getFacet(IKeyword.DATA) != null) {
s.append(getLiteral(IKeyword.DATA));
} else {
Expand Down
5 changes: 3 additions & 2 deletions msi.gama.core/src/msi/gama/outputs/MonitorOutput.java
View file
Expand Up @@ -148,7 +148,8 @@ private void setColor(final IExpression facet) {
/**
* Sets the color.
*
* @param gamaColor the new color
* @param gamaColor
* the new color
*/
public void setColor(final GamaColor gamaColor) {
color = gamaColor;
Expand Down Expand Up @@ -272,7 +273,7 @@ public void saveHistory() {
monitorFolder + "/" + "monitor_" + getName() + "_cycle_" + getScope().getClock().getCycle() + ".csv";
file = FileUtils.constructAbsoluteFilePath(getScope(), file, false);
try (final BufferedWriter bw = new BufferedWriter(new FileWriter(file));
final CsvWriter w = new CsvWriter(bw, CsvWriter.Letters.COMMA)) {
final CsvWriter w = new CsvWriter(bw)) {
for (final Object o : history) {
String[] strings = null;
if (o instanceof Number) {
Expand Down

0 comments on commit fbe60ca

Please sign in to comment.