[Genstar] extension prototype that provide new generate statement

espacedev.gaml.extensions.genstar/META-INF/MANIFEST.MF

@@ -0,0 +1,21 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Genstar
+Bundle-SymbolicName: espacedev.gaml.extensions.genstar;singleton:=true
+Bundle-Version: 1.0.0.qualifier
+Automatic-Module-Name: espacedev.gaml.extensions.genstar
+Bundle-RequiredExecutionEnvironment: JavaSE-15
+Require-Bundle: msi.gama.core;bundle-version="1.8.1",
+ msi.gama.ext;bundle-version="1.8.1"
+Bundle-ClassPath: lib/genstar-core-1.0.4.jar,
+ lib/genstar-gospl-1.0.4.jar,
+ lib/genstar-spll-1.0.4.jar,
+ lib/poi-ooxml-3.9.jar,
+ .,
+ lib/log4j-api-2.7.jar,
+ lib/log4j-core-2.7.jar,
+ lib/opencsv-2.3.jar
+Export-Package: espacedev.gaml.extensions.genstar.generator,
+ espacedev.gaml.extensions.genstar.statement,
+ espacedev.gaml.extensions.genstar.type,
+ espacedev.gaml.extensions.genstar.utils

espacedev.gaml.extensions.genstar/build.properties

@@ -0,0 +1,13 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .,\
+               lib/genstar-core-1.0.4.jar,\
+               lib/genstar-gospl-1.0.4.jar,\
+               lib/genstar-spll-1.0.4.jar,\
+               lib/poi-ooxml-3.9.jar,\
+               plugin.xml,\
+               models/,\
+               lib/log4j-api-2.7.jar,\
+               lib/log4j-core-2.7.jar,\
+               lib/opencsv-2.3.jar

espacedev.gaml.extensions.genstar/models/Genstar Synthesis/.project

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>Genstar Synthesis</name>
+	<comment>genstar plugin</comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.xtext.ui.shared.xtextBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.xtext.ui.shared.xtextNature</nature>
+		<nature>msi.gama.application.gamaNature</nature>
+		<nature>msi.gama.application.pluginNature</nature>
+	</natures>
+</projectDescription>

espacedev.gaml.extensions.genstar/models/Genstar Synthesis/includes/Age & Sexe-Tableau 1.csv

@@ -0,0 +1,23 @@
+;Hommes;Femmes;Ensemble;;;
+Moins de 5 ans;19748;18403;38151;;;
+5 à 9 ans;19145;17923;37068;;;
+10 à 14 ans;19026;18409;37435;;;
+15 à 19 ans;21329;20854;42183;;;
+20 à 24 ans;23064;23802;46866;;;
+25 à 29 ans;20765;21101;41866;;;
+30 à 34 ans;18899;19152;38051;;;
+35 à 39 ans;18725;19396;38121;;;
+40 à 44 ans;20043;20892;40935;;;
+45 à 49 ans;19951;21356;41307;;;
+50 à 54 ans;20222;21843;42065;;;
+55 à 59 ans;19209;20972;40181;;;
+60 à 64 ans;17977;19848;37825;;;
+65 à 69 ans;12139;14467;26606;;;
+70 à 74 ans;9417;12078;21495;;;
+75 à 79 ans;8028;12465;20493;;;
+80 à 84 ans;6126;10795;16922;;;
+85 à 89 ans;3426;7261;10687;;;
+90 à 94 ans;1064;3052;4116;;;
+95 à 99 ans;153;779;932;;;
+100 ans ou plus;13;168;181;;;
+Ensemble;298469;325017;623486;;;

espacedev.gaml.extensions.genstar/models/Genstar Synthesis/models/minimal gosp - rouen.gaml

@@ -0,0 +1,55 @@
+/**
+* Name: minimal generation of synthetic population (gosp) exemple 
+* Author: chapuisk
+* Description: Provide minimal exemple on how to use aggregated statistical data (x2 attributes) to generate a synthtic population in Gama
+* Tags: Tag1, Tag2, TagN
+*/
+
+model minimal_gosp
+
+global {
+
+	init {	
+		generate species:people number: 10000 
+			from:[csv_file("../includes/Age & Sexe-Tableau 1.csv",";")] 
+			attributes:["Age"::["Moins de 5 ans", "5 à 9 ans", "10 à 14 ans", "15 à 19 ans", "20 à 24 ans", 
+					  				"25 à 29 ans", "30 à 34 ans", "35 à 39 ans", "40 à 44 ans", "45 à 49 ans", 
+									"50 à 54 ans", "55 à 59 ans", "60 à 64 ans", "65 à 69 ans", "70 à 74 ans", "75 à 79 ans", 
+									"80 à 84 ans", "85 à 89 ans", "90 à 94 ans", "95 à 99 ans", "100 ans ou plus"],
+						"Sexe"::["Hommes", "Femmes"]];
+
+	}
+}
+
+species people {
+	int Age;
+	string Sexe;
+
+	aspect default { 
+		draw circle(2) color: #red border: #black;
+	}
+}
+
+experiment Rouentemplate type: gui {
+	output {
+		display map {
+			species people;
+		}
+
+		display c {
+			chart "ages" type: histogram {
+				loop i from: 0 to: 110 {
+					data ""+i value: people count(each.Age = i);
+				}
+			}
+		}
+
+		display s {
+			chart "sex" type: pie {
+				loop se over: ["Hommes", "Femmes"] {
+					data se value: people count(each.Sexe = se);
+				}
+			}
+		}
+	}
+}

espacedev.gaml.extensions.genstar/plugin.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?eclipse version="3.4"?>
+<plugin>
+   <extension
+         point="gaml.extension">
+   </extension>
+</plugin>

espacedev.gaml.extensions.genstar/src/espacedev/gaml/extensions/genstar/generator/FileBasedGenerator.java

@@ -0,0 +1,248 @@
+package espacedev.gaml.extensions.genstar.generator;
+
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+
+import core.configuration.GenstarConfigurationFile;
+import core.configuration.dictionary.AttributeDictionary;
+import core.metamodel.IPopulation;
+import core.metamodel.attribute.Attribute;
+import core.metamodel.attribute.AttributeFactory;
+import core.metamodel.entity.ADemoEntity;
+import core.metamodel.io.GSSurveyWrapper;
+import core.metamodel.value.IValue;
+import core.util.data.GSEnumDataType;
+import core.util.excpetion.GSIllegalRangedData;
+import core.util.random.GenstarRandom;
+import espacedev.gaml.extensions.genstar.statement.GenerateStatement;
+import espacedev.gaml.extensions.genstar.utils.GenStarGamaUtils;
+import gospl.GosplPopulation;
+import gospl.algo.IGosplConcept.EGosplAlgorithm;
+import gospl.algo.sr.ds.DirectSamplingAlgo;
+import gospl.distribution.GosplContingencyTable;
+import gospl.distribution.GosplInputDataManager;
+import gospl.distribution.exception.IllegalControlTotalException;
+import gospl.distribution.exception.IllegalDistributionCreation;
+import gospl.distribution.matrix.INDimensionalMatrix;
+import gospl.distribution.matrix.coordinate.ACoordinate;
+import gospl.generator.DistributionBasedGenerator;
+import gospl.io.exception.InvalidSurveyFormatException;
+import gospl.io.util.ReadDictionaryUtils;
+import gospl.sampler.ISampler;
+import gospl.sampler.sr.GosplBasicSampler;
+import msi.gama.metamodel.agent.IAgent;
+import msi.gama.runtime.IScope;
+import msi.gama.runtime.exceptions.GamaRuntimeException;
+import msi.gama.util.GamaMapFactory;
+import msi.gama.util.IList;
+import msi.gama.util.IMap;
+import msi.gama.util.file.GamaCSVFile;
+import msi.gaml.statements.Arguments;
+import msi.gaml.types.IType;
+import msi.gaml.types.Types;
+import msi.gaml.variables.IVariable;
+
+/**
+ * 
+ * Genstar translation of Gama Delegate
+ * 
+ * @author kevinchapuis
+ *
+ */
+public class FileBasedGenerator implements IGenstarGenerator {
+
+	// SINGLETONG
+	private static final FileBasedGenerator INSTANCE = new FileBasedGenerator();
+	public static FileBasedGenerator getInstance() {return INSTANCE;}
+
+	@SuppressWarnings("rawtypes")
+	final IType type;
+
+	@SuppressWarnings("unchecked")
+	private FileBasedGenerator() { this.type = Types.LIST.of(Types.FILE); }
+
+	@SuppressWarnings("rawtypes")
+	@Override
+	public IType sourceType() { return type; }
+
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	@Override
+	public boolean sourceMatch(IScope scope, Object source) {
+		return source instanceof IList && ((IList) source).stream(scope).allMatch(csv -> csv instanceof GamaCSVFile);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public void generate(IScope scope, List<Map<String, Object>> inits, Integer max, 
+			Object source, Object attributes, Object algo, 
+			Arguments init, GenerateStatement generateStatement) {
+		IAgent executor = scope.getAgent();
+		msi.gama.metamodel.population.IPopulation<? extends IAgent> gamaPop = executor
+				.getPopulationFor(generateStatement.getDescription().getSpeciesContext().getName());
+
+		// --------
+		// 1. Infer the type of data for each attributes (based on gaml type and values given)
+		// -------
+		IMap<String,IList<String>> atts = (IMap<String,IList<String>>) attributes;
+		List<Attribute<? extends IValue>> gsAttributes = new ArrayList<>();
+		for (String a : atts.getKeys()) {
+
+			// Trying to infer the type of data given
+			@SuppressWarnings("rawtypes")
+			IType gamaT = gamaPop.getVar(a).getType();
+			GSEnumDataType gsT = ReadDictionaryUtils.detectIsRange(atts.get(a)) ? 
+					GSEnumDataType.Range : GenStarGamaUtils.toDataType(gamaT,false);
+
+			Attribute<? extends IValue> newAttribute = null;
+			try {
+				 newAttribute = AttributeFactory.getFactory().createAttribute(a, gsT, atts.get(a));
+			} catch (GSIllegalRangedData e) {
+				e.printStackTrace();
+			}
+			if (newAttribute != null) {gsAttributes.add(newAttribute);}
+		}
+
+		// --------
+		// 1. Infer the type of data in files - contingency, frequency or sample
+		// --------
+		IList<GamaCSVFile> fileSources = (IList<GamaCSVFile>) source;
+		List<GSSurveyWrapper> gsSurveys = fileSources.stream(scope)
+				.map(s -> GenStarGamaUtils.toSurveyWrapper(scope, s, gsAttributes))
+				.collect(Collectors.toList());
+
+		// Set Genstar random engine to be the one of Gama for seed purpose consistancy !
+		GenstarRandom.setInstance(scope.getRandom().getGenerator());
+
+		////////////////////////////////////////////////////////////////////////
+		// Setup Gen* data
+		////////////////////////////////////////////////////////////////////////
+
+		// Create a basic empty Genstar population
+		IPopulation<ADemoEntity, Attribute<? extends IValue>> population = new GosplPopulation();
+
+		// TODO : retrieve all the required configuration to build a Genstar configuration
+		Path baseDirectory = FileSystems.getDefault().getPath(".");		
+
+		GenstarConfigurationFile confFile = new GenstarConfigurationFile();
+		confFile.setBaseDirectory(baseDirectory);
+		confFile.setSurveyWrappers(gsSurveys);
+		confFile.setDictionary(new AttributeDictionary(gsAttributes));
+
+		////////////////////////////////////////////////////////////////////////
+		// Gospl generation
+		////////////////////////////////////////////////////////////////////////
+
+		GosplInputDataManager gdb = new GosplInputDataManager(confFile);
+
+        final EGosplAlgorithm gsAlgo = algo==null ? EGosplAlgorithm.DS : GenStarGamaUtils.toGosplAlgorithm(algo.toString());
+
+        switch (gsAlgo.concept) {
+			case CO:
+				break;
+			case MIXTURE:
+				throw new UnsupportedOperationException("Mixture population synthesis have not yet been ported from API to plugin ! "
+					+ "request dev at https://github.com/ANRGenstar/genstar.gamaplugin ;)");
+			case MULTILEVEL: throw new UnsupportedOperationException("I'll do it asap");
+			case SR:
+			default:
+				// Build the n-dimensional matrix from raw data
+				INDimensionalMatrix<Attribute<? extends IValue>, IValue, Double> distribution = manageRawData(scope, gdb);
+				ISampler<ACoordinate<Attribute<? extends IValue>, IValue>> sampler = null;
+				switch (gsAlgo) {
+					case HS: 
+						break;
+					case DS:
+					default:
+						try {
+							sampler = new DirectSamplingAlgo().inferSRSampler(distribution, new GosplBasicSampler());
+						} catch (final IllegalDistributionCreation e1) {
+							throw GamaRuntimeException.error("Error of distribution creation in infering the sampler for "+gsAlgo.name
+									+" SR Based algorithm. "+e1.getMessage(), scope);
+						}
+						break; 
+				}
+				population = new DistributionBasedGenerator(sampler).generate(inferPopulationSize(max, gdb));
+				break;
+        }
+
+		////////////////////////////////////////////////////////////////////////
+		// Transpose Gen* entities to Gama species
+		////////////////////////////////////////////////////////////////////////
+
+        if (max > 0 && max < population.size()) scope.getRandom().shuffleInPlace(population);
+
+        for (final ADemoEntity e : population) {
+        	@SuppressWarnings("rawtypes")
+			final Map map = GamaMapFactory.create();
+        	for (final Attribute<? extends IValue> attribute : gsAttributes) {  
+        		IVariable var = gamaPop.getVar(attribute.getAttributeName());
+        		map.put(attribute.getAttributeName(), 
+        				GenStarGamaUtils.toGAMAValue(scope, e.getValueForAttribute(attribute), true, var.getType()));
+        	}
+        	generateStatement.fillWithUserInit(scope, map);
+    		inits.add(map);
+        }
+
+	}
+
+	// -------------------------------------------------------------------- //
+
+	// SR Utils
+
+	/**
+	 * Construct a n-dimensional matrix based on raw data
+	 * 
+	 * @param scope
+	 * @param gdb
+	 * @return
+	 */
+	public static INDimensionalMatrix<Attribute<? extends IValue>, IValue, Double> manageRawData(IScope scope, GosplInputDataManager gdb) {
+		try {
+			gdb.buildDataTables();  // Load and read input data
+		} catch (final RuntimeException | InvalidFormatException | IOException | InvalidSurveyFormatException e) {
+			throw GamaRuntimeException.error("Error in building dataTable for the IS algorithm. "+e.getMessage(), scope);
+		}
+
+		INDimensionalMatrix<Attribute<? extends IValue>, IValue, Double> distribution = null;
+		try {
+			distribution = gdb.collapseDataTablesIntoDistribution(); // Build a distribution from input data
+		} catch (final IllegalDistributionCreation e1) {
+			throw GamaRuntimeException.error("Error of distribution creation in collapsing DataTable into distibution. "+e1.getMessage(), scope);
+		} catch (final IllegalControlTotalException e1) {
+			throw GamaRuntimeException.error("Error of control in collapsing DataTable into distibution. "+e1.getMessage(), scope);
+		}
+		return distribution;
+	}
+
+	/**
+	 * Try to find a good fit in the data to decide the proper number of synthetic population size, i.e. in the case there is contingencies
+	 * 
+	 * @param requestedSize
+	 * @param gdb
+	 * @return
+	 */
+	public static int inferPopulationSize(int requestedSize, GosplInputDataManager gdb) {
+		// DEFINE THE POPULATION SIZE
+		if (requestedSize < 0) {
+			int min = Integer.MAX_VALUE;
+			for (INDimensionalMatrix<Attribute<? extends IValue>,IValue,? extends Number> mat: gdb.getRawDataTables()) {
+				if (mat instanceof GosplContingencyTable) {
+					GosplContingencyTable cmat = (GosplContingencyTable) mat;
+					min = Math.min(min, cmat.getMatrix().values().stream().mapToInt(v -> v.getValue()).sum());
+				}
+			}
+			if (min < Integer.MAX_VALUE) {
+				requestedSize = min;
+			} else requestedSize = 1;
+		}
+		return requestedSize <= 0 ? 1 : requestedSize;
+	}
+
+}