Permalink
Browse files

[#83]: Automatic Evaluation of Exploration Candidates

Added the necessary capabilities

Closes #83

Task-Url: http://github.com/roberthilbrich/assist-public/issues/83
  • Loading branch information...
RobertHilbrich committed Jul 11, 2018
1 parent 3d21e48 commit f2de25c3904575b1c8f4161dd4ae8bd52dc22855
@@ -2,6 +2,7 @@ package ch.hilbri.assist.dse.tests.basics
import ch.hilbri.assist.dse.evaluation.Evaluation
import ch.hilbri.assist.dse.tests.AbstractDSETest
import ch.hilbri.assist.dse.ui.handlers.EvalDesignSpace.Mode
import ch.hilbri.assist.model.AssistModel
import java.io.File
import java.nio.file.Files
@@ -43,7 +44,7 @@ class VariancePointRemovalTests extends AbstractDSETest {
val r = resourceHelper.resource(input)
val assistModel = r.contents.get(0) as AssistModel
val evalJob = new Evaluation(assistModel)
val evalJob = new Evaluation(assistModel, Mode.FEASIBILITY_ONLY)
evalJob.run
val explorationCandidates = evalJob.explorationResults
@@ -1,8 +1,12 @@
package ch.hilbri.assist.dse.evaluation
import ch.hilbri.assist.dse.results.ExplorationResult
import ch.hilbri.assist.dse.ui.handlers.EvalDesignSpace
import ch.hilbri.assist.dse.ui.handlers.EvalDesignSpace.Mode
import ch.hilbri.assist.mapping.analysis.ResultsAnalysis
import ch.hilbri.assist.mapping.solver.AssistMappingSolver
import ch.hilbri.assist.mapping.solver.exceptions.BasicConstraintsException
import ch.hilbri.assist.model.AbstractMetric
import ch.hilbri.assist.model.AssistModel
import ch.qos.logback.classic.Level
import ch.qos.logback.classic.LoggerContext
@@ -14,53 +18,61 @@ import org.slf4j.LoggerFactory
class Evaluation {
AssistModel assistModel
Logger logger = LoggerFactory.getLogger(this.class)
@Accessors(PUBLIC_GETTER) List<ExplorationResult> explorationResults = newArrayList()
new(AssistModel input) {
logger.info('''************************************************''')
logger.info(''' ASSIST Design Space Exploration''')
logger.info('''************************************************''')
assistModel = input
}
def run() {
/* Go through all exploration candidates */
for (i : 0 ..< assistModel.explorationCandidates.size) {
val explorationCandidateModel = EcoreUtil.copy(assistModel)
val candidate = explorationCandidateModel.explorationCandidates.get(i)
logger.info('''Exploring Candidate "«candidate.name»"''')
/* Remove the variance points in the hardware architecture, which are not needed */
logger.info(''' - Preparing hardware model (removing alternatives)''')
for (box : explorationCandidateModel.allBoxes) {
for (boardAlternative : box.boardAlternatives) {
for (alternative : boardAlternative.alternatives) {
if (candidate.boardAlternatives.contains(alternative)) {
// The candidate wants these boards
box.boards.addAll(alternative.boards)
}
}
}
box.boardAlternatives.clear
}
/* Remove the variance points in the software architecture, which are not needed in the current candidate */
logger.info(''' - Preparing software model (removing alternatives)''')
for (applicationAlternative : explorationCandidateModel.applicationAlternatives) {
for (alternative : applicationAlternative.alternatives) {
if (candidate.applicationAlternatives.contains(alternative)) {
// The candidate wants these applications
explorationCandidateModel.applications.addAll(alternative.applications)
}
}
}
explorationCandidateModel.applicationAlternatives.clear
AssistModel assistModel
Logger logger = LoggerFactory.getLogger(this.class)
@Accessors(PUBLIC_GETTER) List<ExplorationResult> explorationResults = newArrayList()
Mode mode
AbstractMetric metric
new(AssistModel input, Mode mo) {
this(input, mo, null)
}
new(AssistModel input, Mode mo, AbstractMetric me) {
logger.info('''************************************************''')
logger.info(''' ASSIST Design Space Exploration''')
logger.info('''************************************************''')
assistModel = input
mode = mo
metric = me
}
def run() {
/* Go through all exploration candidates */
for (i : 0 ..< assistModel.explorationCandidates.size) {
val explorationCandidateModel = EcoreUtil.copy(assistModel)
val candidate = explorationCandidateModel.explorationCandidates.get(i)
logger.info('''Exploring Candidate "«candidate.name»"''')
/* Remove the variance points in the hardware architecture, which are not needed */
logger.info(''' - Preparing hardware model (removing alternatives)''')
for (box : explorationCandidateModel.allBoxes) {
for (boardAlternative : box.boardAlternatives) {
for (alternative : boardAlternative.alternatives) {
if (candidate.boardAlternatives.contains(alternative)) {
// The candidate wants these boards
box.boards.addAll(alternative.boards)
}
}
}
box.boardAlternatives.clear
}
/* Remove the variance points in the software architecture, which are not needed in the current candidate */
logger.info(''' - Preparing software model (removing alternatives)''')
for (applicationAlternative : explorationCandidateModel.applicationAlternatives) {
for (alternative : applicationAlternative.alternatives) {
if (candidate.applicationAlternatives.contains(alternative)) {
// The candidate wants these applications
explorationCandidateModel.applications.addAll(alternative.applications)
}
}
}
explorationCandidateModel.applicationAlternatives.clear
/* Remove the variance points in the restrictions */
logger.info(''' - Preparing the restrictions model (removing alternatives)''')
for (restrictionAlternative : explorationCandidateModel.restrictionAlternatives) {
@@ -69,46 +81,61 @@ class Evaluation {
// The candidate wants these restrictions
explorationCandidateModel.colocalityRelations.addAll(alternative.colocalityRelations)
explorationCandidateModel.dislocalityRelations.addAll(alternative.dislocalityRelations)
explorationCandidateModel.dissimilarityRelations.addAll(alternative.dissimilarityRelations)
explorationCandidateModel.dissimilarityRelations.addAll(alternative.dissimilarityRelations)
}
}
}
explorationCandidateModel.restrictionAlternatives.clear
/* Remove the exploration candidates from the model */
logger.info(''' - Preparing the list of exploration candidates (removing all candidates)''')
explorationCandidateModel.explorationCandidates.clear
/* Now we have a pristine model */
logger.info(''' - Checking mapping feasibility''')
/* We do not want see all the output from the mapping, so we mute it */
val loggerContext = LoggerFactory.getILoggerFactory() as LoggerContext
val mappingLogger = loggerContext.getLogger("ch.hilbri.assist.mapping")
mappingLogger.setLevel(Level.OFF)
val mappingSolver = new AssistMappingSolver(explorationCandidateModel)
try {
mappingSolver.runInitialization
mappingSolver.solverMaxSolutions = 1
mappingSolver.runConstraintGeneration
mappingSolver.runSolutionSearch
mappingSolver.createSolutions
} catch (BasicConstraintsException e) {
logger.info(''' - Mapping failed (conflicting constraint "«e.constraintName»")''')
}
/* Of course, we have to re-enable the logging output from the mapping part */
mappingLogger.setLevel(Level.DEBUG)
if (mappingSolver.results !== null && mappingSolver.results.size > 0) {
logger.info('''Candidate "«candidate.name»" seems feasible''')
explorationResults.add(new ExplorationResult(candidate, explorationCandidateModel, true))
}
else {
logger.info('''Candidate "«candidate.name»" seems NOT feasible''')
explorationResults.add(new ExplorationResult(candidate, explorationCandidateModel, false))
}
}
}
/* Now we have a pristine model */
logger.info(''' - Checking mapping feasibility''')
/* We do not want see all the output from the mapping, so we mute it */
val loggerContext = LoggerFactory.getILoggerFactory() as LoggerContext
val mappingLogger = loggerContext.getLogger("ch.hilbri.assist.mapping")
mappingLogger.setLevel(Level.OFF)
val mappingSolver = new AssistMappingSolver(explorationCandidateModel)
try {
mappingSolver.runInitialization
mappingSolver.solverMaxSolutions = 1
mappingSolver.runConstraintGeneration
mappingSolver.runSolutionSearch
mappingSolver.createSolutions
} catch (BasicConstraintsException e) {
logger.info(''' - Mapping failed (conflicting constraint "«e.constraintName»")''')
}
/* Of course, we have to re-enable the logging output from the mapping part */
mappingLogger.setLevel(Level.DEBUG)
if (mappingSolver.results !== null && mappingSolver.results.size > 0) {
// Candidate is feasible and we want scoring too
if (mode == EvalDesignSpace.Mode.FEASIBILITY_AND_SCORING) {
val result = mappingSolver.results.head
logger.info(''' - Scoring the result''')
if (metric === null) {
logger.info(''' Scoring is not possible, because we did not get a valid metric''')
} else {
ResultsAnalysis.evaluate(#[result], #[metric])
explorationResults.add(new ExplorationResult(candidate, explorationCandidateModel, true, result.absoluteTotalScore))
}
} // Candidate is not feasible and we do not want any scoring
else {
explorationResults.add(new ExplorationResult(candidate, explorationCandidateModel, true, null))
}
logger.info('''Candidate "«candidate.name»" seems feasible''')
} else {
explorationResults.add(new ExplorationResult(candidate, explorationCandidateModel, false, null))
logger.info('''Candidate "«candidate.name»" seems NOT feasible''')
}
}
}
}
@@ -1,6 +1,8 @@
package ch.hilbri.assist.dse.evaluation
import ch.hilbri.assist.dse.ui.handlers.EvalDesignSpace.Mode
import ch.hilbri.assist.dse.ui.multipageeditor.MultiPageEditor
import ch.hilbri.assist.model.AbstractMetric
import ch.hilbri.assist.model.AssistModel
import org.eclipse.core.runtime.IProgressMonitor
import org.eclipse.core.runtime.Status
@@ -14,7 +16,7 @@ class GuiEvaluationJob extends Job {
MultiPageEditor multiPageEditor
Evaluation evaluation // private Logger logger = LoggerFactory.getLogger(GuiEvaluationJob)
new(String name, URI modelURI, MultiPageEditor editor) {
new(String name, URI modelURI, MultiPageEditor editor, Mode mode, AbstractMetric metric) {
super(name);
/* Who is asking to get the results back? */
@@ -25,7 +27,7 @@ class GuiEvaluationJob extends Job {
val resource = rs.getResource(modelURI, true)
val assistModel = resource.contents.get(0) as AssistModel
evaluation = new Evaluation(assistModel)
evaluation = new Evaluation(assistModel, mode, metric)
}
override protected run(IProgressMonitor monitor) {
@@ -8,4 +8,5 @@ import ch.hilbri.assist.model.ExplorationCandidate
ExplorationCandidate candidate
AssistModel model
boolean isFeasible
Double score
}
@@ -0,0 +1,126 @@
package ch.hilbri.assist.dse.ui.evaluatesettings;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.jface.dialogs.TitleAreaDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Combo;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Group;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
import ch.hilbri.assist.dse.ui.handlers.EvalDesignSpace.Mode;
import ch.hilbri.assist.mapping.analysis.metrics.builtin.RandomScore;
import ch.hilbri.assist.model.AbstractMetric;
public class EvaluateDesignSpaceSettingsDialog extends TitleAreaDialog {
private Label lblSelectMetric;
private Combo cbxSelectMetric;
private Button btnFeasibilityCheckOnly;
private Button btnFeasibilityAndScoring;
private List<AbstractMetric> availableMetrics = new ArrayList<AbstractMetric>();
private Mode currentMode = Mode.FEASIBILITY_ONLY;
private AbstractMetric currentMetric;
public EvaluateDesignSpaceSettingsDialog(Shell parentShell) {
super(parentShell);
/* Not all metrics make sense from a DSE perspective -
* most of the mapping metrics are more concerned with
* comparing different results with each other, while the
* DSE only tries to calculate one result for each product */
availableMetrics.add(new RandomScore());
}
/**
* Create contents of the dialog.
*
* @param parent
*/
@Override
protected Control createDialogArea(Composite parent) {
setMessage("Please set the parameters for the design space exploration");
setTitle("Design Space Exploration");
getShell().setText("Design Space Exploration");
Composite area = (Composite) super.createDialogArea(parent);
area.setLayout(new GridLayout(1, false));
Group grpMode = new Group(area, SWT.NONE);
grpMode.setLayout(new GridLayout(1, false));
grpMode.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false, 1, 1));
grpMode.setText("Mode");
btnFeasibilityCheckOnly = new Button(grpMode, SWT.RADIO);
btnFeasibilityCheckOnly.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
currentMode = Mode.FEASIBILITY_ONLY;
lblSelectMetric.setEnabled(false);
cbxSelectMetric.setEnabled(false);
cbxSelectMetric.removeAll();
}
});
btnFeasibilityCheckOnly.setSelection(true);
btnFeasibilityCheckOnly.setText("Feasibility check only");
btnFeasibilityAndScoring = new Button(grpMode, SWT.RADIO);
btnFeasibilityAndScoring.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
currentMode = Mode.FEASIBILITY_AND_SCORING;
lblSelectMetric.setEnabled(true);
cbxSelectMetric.setEnabled(true);
for (AbstractMetric metric : availableMetrics) {
cbxSelectMetric.add(metric.getName());
}
cbxSelectMetric.select(0);
cbxSelectMetric.notifyListeners(SWT.Selection, new Event());
}
});
GridData gd_btnFeasibilityAndScoring = new GridData(SWT.LEFT, SWT.CENTER, false, false, 1, 1);
gd_btnFeasibilityAndScoring.verticalIndent = 10;
btnFeasibilityAndScoring.setLayoutData(gd_btnFeasibilityAndScoring);
btnFeasibilityAndScoring.setText("Feasibility and Scoring");
lblSelectMetric = new Label(grpMode, SWT.NONE);
lblSelectMetric.setEnabled(false);
GridData gd_lblSelectMetric = new GridData(SWT.LEFT, SWT.CENTER, false, false, 1, 1);
gd_lblSelectMetric.horizontalIndent = 20;
lblSelectMetric.setLayoutData(gd_lblSelectMetric);
lblSelectMetric.setText("Please select metric for scoring:");
cbxSelectMetric = new Combo(grpMode, SWT.READ_ONLY);
cbxSelectMetric.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
currentMetric = availableMetrics.get(cbxSelectMetric.getSelectionIndex());
}
});
cbxSelectMetric.setEnabled(false);
GridData gd_cbxSelectMetric = new GridData(SWT.FILL, SWT.CENTER, true, false, 1, 1);
gd_cbxSelectMetric.horizontalIndent = 20;
cbxSelectMetric.setLayoutData(gd_cbxSelectMetric);
return area;
}
public Mode getMode() {
return currentMode;
}
public AbstractMetric getScoringMetric() {
return currentMetric;
}
}
Oops, something went wrong.

0 comments on commit f2de25c

Please sign in to comment.