Skip to content
Browse files

-add: CompleteComparisonExport

-change: MUTEXLexer
  • Loading branch information...
1 parent cae4c49 commit f1a3e6d94b12e8959348b61cc6d939f59ce4e1a4 @yas4891 committed Sep 3, 2012
Showing with 11,673 additions and 1,594 deletions.
  1. +6 −0 CTokenizer/.classpath
  2. +23 −0 CTokenizer/.project
  3. +12 −0 CTokenizer/.settings/org.deved.antlride.core.prefs
  4. +35 −1 CTokenizer/LexerHelper.cs
  5. +10,595 −1,464 CTokenizer/MutexCLexer.cs
  6. +184 −31 CTokenizer/MutexCLexer.g
  7. +108 −18 CTokenizer/MutexCLexer.tokens
  8. +4 −0 GSTAppLogic/GSTAppLogic.csproj
  9. +35 −0 GSTAppLogic/ext/AppHelper.cs
  10. +7 −1 GSTAppLogic/ext/LexerExtensions.cs
  11. +42 −0 GSTAppLogic/templating/TemplatingHelper.cs
  12. +46 −0 GSTAppLogic/test/templating/TemplatingHelperTest.cs
  13. +2 −0 GSTEvaluation/GSTEvaluation.csproj
  14. +44 −9 GSTEvaluation/Program.cs
  15. +49 −0 GSTEvaluation/bin/Debug/log4net.xml
  16. +39 −34 GSTEvaluation/export/ComparisonExcelExport.cs
  17. +159 −0 GSTEvaluation/export/CompleteComparisonReport.cs
  18. +16 −2 GSTEvaluation/export/ExcelExport.cs
  19. +1 −1 GSTEvaluation/export/FileExport.cs
  20. +4 −2 GSTEvaluation/export/ListResultsExport.cs
  21. +116 −0 GSTEvaluation/export/RuntimeExcelExport.cs
  22. +21 −1 GSTEvaluation/model/ComparisonHistoryModel.cs
  23. +19 −5 GSTEvaluation/model/ComparisonModel.cs
  24. +8 −4 GSTEvaluation/model/EvaluationRunModel.cs
  25. +41 −10 GSTEvaluation/storage/SQLFacade.cs
  26. BIN GSTEvaluation120807(1).vsp
  27. BIN GSTEvaluation120807.vsp
  28. BIN GSTEvaluation120809.vsp
  29. +0 −1 GSTLibrary/test/tile/GSTAlgorithmTest.cs
  30. +9 −1 GSTLibrary/tile/GSTAlgorithm.cs
  31. +1 −0 GSTLibrary/token/GSTToken.cs
  32. +8 −7 GSTTests.VisualState.xml
  33. +5 −0 GreedyStringTiling.sln
  34. +25 −0 Performance1.psess
  35. +0 −1 Tokenizer/TokenType.cs
  36. +9 −1 Tokenizer/TokenWrapper.cs
View
6 CTokenizer/.classpath
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
View
23 CTokenizer/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>CTokenizer</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.dltk.core.scriptbuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.deved.antlride.core.nature</nature>
+ </natures>
+</projectDescription>
View
12 CTokenizer/.settings/org.deved.antlride.core.prefs
@@ -0,0 +1,12 @@
+#Wed Aug 22 20:40:11 CEST 2012
+antlr_core_code_generator_XdbgST=false
+antlr_core_code_generator_append_java_package_to_out_folder=false
+antlr_core_code_generator_debug=false
+antlr_core_code_generator_max_memory=0
+antlr_core_code_generator_out_folder=antlr-generated
+antlr_core_code_generator_out_option=g
+antlr_core_code_generator_profile=false
+antlr_core_code_generator_trace=false
+antlr_core_code_generator_x_max_switch_case_labels=300
+antlr_core_code_generator_x_min_switch_alts=3
+eclipse.preferences.version=1
View
36 CTokenizer/LexerHelper.cs
@@ -13,7 +13,23 @@ namespace CTokenizer
/// </summary>
public static class LexerHelper
{
- internal static readonly Type UsedLexer = typeof(CLexer);
+ private static Type usedLexer = typeof(CLexer);
+
+ public static Type UsedLexer
+ {
+ get { return usedLexer; }
+ set
+ {
+ if(null == value)
+ throw new ArgumentNullException("value can not be NULL");
+
+ if(!(typeof(Lexer).IsAssignableFrom(value)))
+ throw new ArgumentException("value must be a sub-class of Antlr.Runtime.Lexer");
+
+
+ usedLexer = value;
+ }
+ }
internal static ConstructorInfo UsedConstructor
{
@@ -105,6 +121,24 @@ public static string GetJoinedTokenString(this Lexer lexer)
}
/// <summary>
+ /// returns the tokens with a bunch of debug information
+ /// </summary>
+ /// <param name="lexer"></param>
+ /// <returns></returns>
+ public static IEnumerable<string> GetDebugTokenStrings(this Lexer lexer)
+ {
+ var lexerType = lexer.GetType();
+
+ return lexer.GetTokens().Select(
+ token => string.Format("[{0:000}:{1:00}] {2}\t\t- {3}",
+ token.Line,
+ token.CharPositionInLine,
+ token.Type.GetTokenName(lexerType),
+ token.Text))
+ .ToList();
+ }
+
+ /// <summary>
/// joins the name of strings with whitespaces
/// </summary>
/// <param name="lexer"></param>
View
12,059 CTokenizer/MutexCLexer.cs
10,595 additions, 1,464 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
215 CTokenizer/MutexCLexer.g
@@ -1,62 +1,215 @@
+/*
+ * this grammar is used to tokenize C/C++ source code.
+ * In contrast to normal lexers, which try to aim for correctness of the
+ * resulting token stream, this lexer is used to fight plagiarism in source
+ * code.
+ * This lexer is used at university to make plagiarism more difficult between students
+ * who have been assigned the same task during CS courses.
+ *
+ * To fight plagiarism it merges different expressions that have the same results into the same
+ * token.
+ * e.g.
+ i++
+ ++i
+ i += 1
+ i = i + 1
+ * those four expressions all create the same token ==> INCREMENT
+ *
+ */
+
lexer grammar MutexCLexer;
options {
language = CSharp3;
filter = true;
- k = 2;
}
-
-
+
+// ignore multi line comment
COMMENT
: '/*' (options {greedy=false;} : . )* '*/'
{ $channel = Hidden;}
;
-
+// ignore single line comment
LINE_COMMENT
-
+// : '//' ~('\n'|'\r')* ('\r' | '\n')+ {$channel= Hidden;}
: '//' ~('\n'|'\r')* '\r'? '\n' {$channel= Hidden;}
-
;
/* */
-FUNCTION_DEFINITION
- : DATATYPE IDENTIFIER '(' PARAMETER_LIST ')';
+// ignore pre-processor as well
+LINE_COMMAND
+// : '#' ~('\n'|'\r')* ('\r' |'\n')+ {$channel=Hidden;}
+ : '#' ~('\n'|'\r')* '\r'? '\n' {$channel=Hidden;}
+ ;
-FUNCTION_CALL : IDENTIFIER '(' PARAMETER (', ' PARAMETER)* ')';
+FOR_LOOP :
+ FOR WS LPARENTHESIS (~')')* RPARENTHESIS;
+
+WHILE_LOOP :
+ WHILE WS LPARENTHESIS (~')')* RPARENTHESIS;
+
+COMPARISON:
+ (IDENTIFIER) WS COMPARISONOPERATOR WS (LITERAL);
+
+COMPARISONOPERATOR:
+ (COMPARISONEQUAL | LESSTHANOREQUAL | GREATERTHANOREQUAL);
+
+DECLARATION_ASSIGNMENT:
+ (INTEGER_DATATYPE | POINTER_DATATYPE | FLOAT_DATATYPE) WS IDENTIFIER WS ASSIGN
+ WS (INTEGER_LITERAL | STRING_LITERAL | DATATYPE);
fragment
-PARAMETER
- : IDENTIFIER | STRING_LITERAL;
+DATATYPE:
+ INTEGER_DATATYPE | POINTER_DATATYPE | FLOAT_DATATYPE ;
+
+
+POINTER_DATATYPE: (INTEGER_DATATYPE | FLOAT_DATATYPE | VOID) WS STAR+;
-PARAMETER_LIST
- : DATATYPE IDENTIFIER (',' DATATYPE IDENTIFIER)*;
+FLOAT_DATATYPE : CONST? WS 'float' | 'double';
-CONST_MODIFIER : 'const';
-
-DATATYPE : VOID_DATATYPE | INTEGER_DATATYPE | FLOAT_DATATYPE { Skip();};
-
-POINTER
- : '*'+;
-
-VOID_DATATYPE : 'void';
-FLOAT_DATATYPE : 'float' | 'double';
-
-INTEGER_DATATYPE : (SIGNED_UNSIGNED)? ('short' | 'int' | 'long' | 'char') ;
-
-SIGNED_UNSIGNED : 'signed' | 'unsigned';
+/*
+ * ignore 'const' and 'signed' | 'unsigned' keywords
+ */
+INTEGER_DATATYPE : CONST? WS (SIGNED_UNSIGNED)? WS ('short' | 'int' | 'long' | 'char') ;
-IDENTIFIER
- : LETTER (LETTER | DIGIT)*;
+INCREMENT : IDENTIFIER (PLUSPLUS | WS ADDEQUAL WS '1') | PLUSPLUS IDENTIFIER ;
+fragment
+LITERAL: STRING_LITERAL | INTEGER_LITERAL;
+/*
+ * covers both decimal and hex integer literals
+ */
+INTEGER_LITERAL :
+ DIGIT+ | '0x' HEX_DIGIT+;
+
STRING_LITERAL
- : '"' (~('\\' | '"'))* '"';
-
+ : '"' (~('"'))* '"';
+ // : '"' (~('\\' | '"'))* '"';
+
+
+/*
+ * the elementary tokens need to be below the aggregated tokens
+ * else the aggregated tokens will not work
+ */
+LCURLYBRACE : '{';
+RCURLYBRACE : '}';
+LSQUAREBRACKET : '[';
+RSQUAREBRACKET : ']';
+LPARENTHESIS : '(';
+RPARENTHESIS : ')';
+SCOPE : '::';
+QUESTIONMARK : '?';
+COLON : ':';
+ADDEQUAL : '+=';
+MINUSEQUAL : '-=';
+TIMESEQUAL : '*=';
+DIVIDEEQUAL : '/=';
+MODEQUAL : '%=';
+SHIFTLEFTEQUAL : '<<=';
+SHIFTRIGHTEQUAL : '>>=';
+ANDEQUAL : '&=';
+OREQUAL : '|=';
+XOREQUAL : '^=';
+SHORTCIRCUITOR : '||';
+SHORTCIRCUITAND : '&&';
+BITWISEOR : '|';
+BITWISEAND : '&';
+COMPARISONEQUAL : '==';
+NOTEQUAL : '!=';
+LESSTHANOREQUAL : '<=';
+GREATERTHANOREQUAL : '>=';
+SHIFTLEFT : '<<';
+SHIFTRIGHT : '>>';
+ASSIGN : '=';
+LESSTHAN : '<';
+GREATERTHAN : '>';
+PLUS : '+';
+MINUS : '-';
+STAR : '*';
+DIVIDE : '/';
+MOD : '%';
+PLUSPLUS : '++';
+MINUSMINUS : '--';
+TILDE : '~';
+NOT : '!';
+DOT : '.';
+POINTERTO : '->';
+BREAK : 'break';
+CASE : 'case';
+CATCH : 'catch';
+//CHAR : 'char';
+CLASS : 'class';
+CONST : 'const';
+CONTINUE : 'continue';
+_DEFAULT : 'default';
+DELETE : 'delete';
+DO : 'do';
+//DOUBLE : 'double';
+ELSE : 'else';
+ENUM : 'enum';
+EXTERN : 'extern';
+//FLOAT : 'float';
+FOR : 'for';
+FRIEND : 'friend';
+GOTO : 'goto';
+IF : 'if';
+INLINE : 'inline';
+//INT : 'int';
+//LONG : 'long';
+NEW : 'new';
+PRIVATE : 'private';
+PROTECTED : 'protected';
+PUBLIC : 'public';
+REDECLARED : 'redeclared';
+REGISTER : 'register';
+RETURN : 'return';
+//SHORT : 'short';
+//SIGNED : 'signed';
+SIZEOF : 'sizeof';
+STATIC : 'static';
+STRUCT : 'struct';
+SWITCH : 'switch';
+TEMPLATE : 'template';
+THIS : 'this';
+TRY : 'try';
+TYPEDEF : 'typedef';
+UNION : 'union';
+UNSIGNED : 'unsigned';
+VIRTUAL : 'virtual';
+VOID : 'void';
+VOLATILE : 'volatile';
+WHILE : 'while';
+OPERATOR : 'operator';
+TRUETOK : 'true';
+FALSETOK : 'false';
+BACKSLASH : '\\';
+
+
+// place this below the keywords
+IDENTIFIER
+ : LETTER (LETTER | DIGIT)*;
+
+
+
fragment
-LETTER : 'A'..'Z' | 'a'..'z' | '_';
+LETTER : 'A'..'Z' | 'a'..'z';
+fragment
HEX_DIGIT : DIGIT | 'a'..'f' | 'A'..'F';
fragment
DIGIT : '0'..'9';
+fragment
+WS : (' ' | '\n' | '\r' | '\t')*;
+
+fragment
+CONST_MODIFIER : 'const';
+
+//fragment
+//VOID_DATATYPE : 'void';
+
+
+fragment
+SIGNED_UNSIGNED : 'signed' | 'unsigned';
View
126 CTokenizer/MutexCLexer.tokens
@@ -1,18 +1,108 @@
-COMMENT=4
-CONST_MODIFIER=5
-DATATYPE=6
-DIGIT=7
-FLOAT_DATATYPE=8
-FUNCTION_CALL=9
-FUNCTION_DEFINITION=10
-HEX_DIGIT=11
-IDENTIFIER=12
-INTEGER_DATATYPE=13
-LETTER=14
-LINE_COMMENT=15
-PARAMETER=16
-PARAMETER_LIST=17
-POINTER=18
-SIGNED_UNSIGNED=19
-STRING_LITERAL=20
-VOID_DATATYPE=21
+ADDEQUAL=4
+ANDEQUAL=5
+ASSIGN=6
+BACKSLASH=7
+BITWISEAND=8
+BITWISEOR=9
+BREAK=10
+CASE=11
+CATCH=12
+CLASS=13
+COLON=14
+COMMENT=15
+COMPARISON=16
+COMPARISONEQUAL=17
+COMPARISONOPERATOR=18
+CONST=19
+CONST_MODIFIER=20
+CONTINUE=21
+DATATYPE=22
+DECLARATION_ASSIGNMENT=23
+DEFAULT=24
+DELETE=25
+DIGIT=26
+DIVIDE=27
+DIVIDEEQUAL=28
+DO=29
+DOT=30
+ELSE=31
+ENUM=32
+EXTERN=33
+FALSETOK=34
+FLOAT_DATATYPE=35
+FOR=36
+FOR_LOOP=37
+FRIEND=38
+GOTO=39
+GREATERTHAN=40
+GREATERTHANOREQUAL=41
+HEX_DIGIT=42
+IDENTIFIER=43
+IF=44
+INCREMENT=45
+INLINE=46
+INTEGER_DATATYPE=47
+INTEGER_LITERAL=48
+LCURLYBRACE=49
+LESSTHAN=50
+LESSTHANOREQUAL=51
+LETTER=52
+LINE_COMMAND=53
+LINE_COMMENT=54
+LITERAL=55
+LPARENTHESIS=56
+LSQUAREBRACKET=57
+MINUS=58
+MINUSEQUAL=59
+MINUSMINUS=60
+MOD=61
+MODEQUAL=62
+NEW=63
+NOT=64
+NOTEQUAL=65
+OPERATOR=66
+OREQUAL=67
+PLUS=68
+PLUSPLUS=69
+POINTERTO=70
+POINTER_DATATYPE=71
+PRIVATE=72
+PROTECTED=73
+PUBLIC=74
+QUESTIONMARK=75
+RCURLYBRACE=76
+REDECLARED=77
+REGISTER=78
+RETURN=79
+RPARENTHESIS=80
+RSQUAREBRACKET=81
+SCOPE=82
+SHIFTLEFT=83
+SHIFTLEFTEQUAL=84
+SHIFTRIGHT=85
+SHIFTRIGHTEQUAL=86
+SHORTCIRCUITAND=87
+SHORTCIRCUITOR=88
+SIGNED_UNSIGNED=89
+SIZEOF=90
+STAR=91
+STATIC=92
+STRING_LITERAL=93
+STRUCT=94
+SWITCH=95
+TEMPLATE=96
+THIS=97
+TILDE=98
+TIMESEQUAL=99
+TRUETOK=100
+TRY=101
+TYPEDEF=102
+UNION=103
+UNSIGNED=104
+VIRTUAL=105
+VOID=106
+VOLATILE=107
+WHILE=108
+WHILE_LOOP=109
+WS=110
+XOREQUAL=111
View
4 GSTAppLogic/GSTAppLogic.csproj
@@ -57,9 +57,12 @@
<Compile Include="app\AppLogicImpl.cs" />
<Compile Include="app\IAppLogic.cs" />
<Compile Include="app\model\ComparisonModel.cs" />
+ <Compile Include="ext\AppHelper.cs" />
<Compile Include="ext\LexerExtensions.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="templating\TemplatingHelper.cs" />
<Compile Include="test\model\ComparisonModelTest.cs" />
+ <Compile Include="test\templating\TemplatingHelperTest.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CTokenizer\CTokenizer.csproj">
@@ -79,6 +82,7 @@
<Name>Tokenizer</Name>
</ProjectReference>
</ItemGroup>
+ <ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
View
35 GSTAppLogic/ext/AppHelper.cs
@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using CTokenizer;
+using GSTLibrary.tile;
+using GSTLibrary.token;
+using Tokenizer;
+
+namespace GSTAppLogic.ext
+{
+ public static class AppHelper
+ {
+
+ /// <summary>
+ /// Compares the two files and returns the Similarity
+ /// </summary>
+ /// <param name="path1"></param>
+ /// <param name="path2"></param>
+ /// <returns></returns>
+ public static Int32 CompareFiles(string path1, string path2)
+ {
+ var tokens1 = LexerHelper.CreateLexer(path1).GetTokenWrappers();
+ var tokens2 = LexerHelper.CreateLexer(path2).GetTokenWrappers();
+
+ var algo = new GSTAlgorithm<GSTToken<TokenWrapper>>(
+ tokens1.ToGSTTokenList<TokenWrapper>(),
+ tokens2.ToGSTTokenList<TokenWrapper>());
+ algo.RunToCompletion();
+
+
+ return algo.Similarity;
+ }
+ }
+}
View
8 GSTAppLogic/ext/LexerExtensions.cs
@@ -1,8 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.IO;
using System.Linq;
using CTokenizer;
+using GSTLibrary.tile;
using GSTLibrary.token;
using Tokenizer;
+using System.Diagnostics;
namespace GSTAppLogic.ext
{
@@ -34,5 +38,7 @@ public static GSTTokenList<GSTToken<TE>> ToGSTTokenList<TE>(this IEnumerable<TE>
{
return new GSTTokenList<GSTToken<TE>>(tokens.Select(token => new GSTToken<TE>(token)));
}
+
+
}
}
View
42 GSTAppLogic/templating/TemplatingHelper.cs
@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace GSTAppLogic.templating
+{
+ /// <summary>
+ /// the purpose of this whole class is to strip source files from the templates.
+ /// This is needed if the professor provided a template to solve the programming problem.
+ /// It will strip the provided template from the final source file
+ /// </summary>
+ public static class TemplatingHelper
+ {
+ /// <summary>
+ /// strips lines from
+ /// </summary>
+ /// <param name="sourcePath"></param>
+ /// <param name="templatePath"></param>
+ /// <returns>path to the temporary stripped file</returns>
+ public static string StripTemplateFromSourceFile(string sourcePath, string templatePath)
+ {
+ var source = File.ReadAllLines(sourcePath).Select(line => line.Trim()).ToList();
+ var template = File.ReadAllLines(templatePath).Select(line => line.Trim()).ToArray();
+
+ foreach(var tmpl in template)
+ {
+ source.Remove(tmpl);
+ }
+
+ var path = Path.GetTempFileName();
+ using(var writer = new StreamWriter(path, false, Encoding.UTF8))
+ {
+ foreach (var line in source)
+ writer.WriteLine(line);
+ }
+
+ return path;
+ }
+ }
+}
View
46 GSTAppLogic/test/templating/TemplatingHelperTest.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using NUnit.Framework;
+using GSTAppLogic.templating;
+
+namespace GSTAppLogic.test.templating
+{
+ [TestFixture]
+ public class TemplatingHelperTest
+ {
+ private string sourcePath;
+ private string templatePath;
+
+ [SetUp]
+ public void SetupFixture()
+ {
+ sourcePath = Path.GetTempFileName();
+ templatePath = Path.GetTempFileName();
+
+ using(var sourceWriter = new StreamWriter(sourcePath, false, Encoding.UTF8))
+ {
+ sourceWriter.WriteLine("void main(void)");
+ sourceWriter.WriteLine("{");
+ sourceWriter.WriteLine("printf(\"%s\", \"Hallo\");");
+ sourceWriter.WriteLine("}");
+ }
+
+ using (var templateWriter = new StreamWriter(templatePath, false, Encoding.UTF8))
+ {
+ templateWriter.WriteLine("printf(\"%s\", \"Hallo\");");
+ }
+ }
+
+
+ [Test]
+ public void TestTemplating()
+ {
+ var stripped = File.ReadAllText(TemplatingHelper.StripTemplateFromSourceFile(sourcePath, templatePath));
+ var split = stripped.Split(new[] {"\r\n"}, StringSplitOptions.RemoveEmptyEntries);
+ Assert.AreEqual("void main(void)", split[0]);
+ }
+ }
+}
View
2 GSTEvaluation/GSTEvaluation.csproj
@@ -61,10 +61,12 @@
</ItemGroup>
<ItemGroup>
<Compile Include="export\ComparisonExcelExport.cs" />
+ <Compile Include="export\CompleteComparisonReport.cs" />
<Compile Include="export\ExcelExport.cs" />
<Compile Include="export\FileExport.cs" />
<Compile Include="export\IExport.cs" />
<Compile Include="export\ListResultsExport.cs" />
+ <Compile Include="export\RuntimeExcelExport.cs" />
<Compile Include="model\ComparisonHistoryModel.cs" />
<Compile Include="model\EvaluationRunModel.cs" />
<Compile Include="model\SourceModel.cs" />
View
53 GSTEvaluation/Program.cs
@@ -7,35 +7,70 @@
using GSTEvaluation.model;
using GSTEvaluation.storage;
using System.Diagnostics;
+using CTokenizer;
+using GSTLibrary.tile;
+using GSTLibrary.token;
+using Tokenizer;
+using log4net;
+using log4net.Config;
namespace GSTEvaluation
{
public class Program
{
- public static readonly string TEST_SUITE_DIRECTORY = @"D:\test\tokenizerEval";
+ private static readonly ILog cLogger = LogManager.GetLogger(typeof(Program));
+
+ public static readonly string TEST_SUITE_DIRECTORY = @"D:\test\MUTEX\tokenizerEval";
public static void Main(string[] args)
{
+ XmlConfigurator.Configure(new FileInfo("log4net.xml"));
+
+
+ /*
+ CompleteComparisonReport.PerformTest("01_01");
+ Console.ReadLine();
+ Environment.Exit(0);
+ /* */
try
{
+ LexerHelper.UsedLexer = typeof (MutexCLexer);
var watch = Stopwatch.StartNew();
var evalModel = new EvaluationRunModel(TEST_SUITE_DIRECTORY);
- Console.WriteLine("evaluation run finished in {0} ms", watch.ElapsedMilliseconds);
+ cLogger.DebugFormat("evaluation run finished in {0} ms", watch.ElapsedMilliseconds);
new ListResultsExport().Run(evalModel);
- new ComparisonExcelExport("comparisons").Run(
- new[]
- {
- new ComparisonHistoryModel("DupV01-S01-13"),
- });
+ File.WriteAllLines(@"D:\test\testfile.token.txt", LexerHelper.CreateLexer(@"D:\test\test.c").GetDebugTokenStrings());
+
+ File.WriteAllLines(@"D:\test\tok1.txt", LexerHelper.CreateLexer(@"D:\test\main-01.c").GetDebugTokenStrings());
+ File.WriteAllLines(@"D:\test\tok2.txt", LexerHelper.CreateLexer(@"D:\test\main-03.c").GetDebugTokenStrings());
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
-
+
+ //if ("y" == decision)
+ {
+
+ //new CompleteComparisonReport("01_01").Run();
+
+
+ new ComparisonExcelExport("comparisons").Run(
+ ComparisonHistoryModel.AllHistories);
+
+ new RuntimeExcelExport("runtime").Run(
+ new[]
+ {
+ new ComparisonHistoryModel("TPLV04-S01-02"),
+ });
+ }
+
+ Console.WriteLine();
+ Console.WriteLine();
Console.WriteLine("Done!");
- Console.ReadLine();
+ Console.WriteLine("Type [y] if you want to create excel spreadsheets");
+ var decision = Console.ReadLine();
}
}
}
View
49 GSTEvaluation/bin/Debug/log4net.xml
@@ -0,0 +1,49 @@
+<log4net>
+ <!-- A1 is set to be a ConsoleAppender -->
+ <appender name="Console" type="log4net.Appender.ConsoleAppender">
+
+ <!-- A1 uses PatternLayout -->
+ <layout type="log4net.Layout.PatternLayout">
+ <conversionPattern value="{%logger}:%line %level: - %message %newline" />
+ </layout>
+ <threshold value="DEBUG"/>
+ </appender>
+
+ <appender name="RollingFileDebug" type="log4net.Appender.RollingFileAppender">
+ <file type="log4net.Util.PatternString">
+ <conversionPattern value="debug.log" />
+ </file>
+
+ <appendToFile value="false" />
+ <maximumFileSize value="100MB" />
+ <maxSizeRollBackups value="5" />
+ <rollingStyle value="Size" />
+
+ <layout type="log4net.Layout.PatternLayout">
+ <conversionPattern value="[%date] {%logger}:%line %level: - %message %newline" />
+ </layout>
+ </appender>
+
+ <appender name="RollingFileDebug2" type="log4net.Appender.RollingFileAppender">
+ <file type="log4net.Util.PatternString">
+ <conversionPattern value="D:/test/debug.log" />
+ </file>
+
+ <appendToFile value="false" />
+ <maximumFileSize value="100MB" />
+ <maxSizeRollBackups value="5" />
+ <rollingStyle value="Size" />
+
+ <layout type="log4net.Layout.PatternLayout">
+ <conversionPattern value="[%date] {%logger}:%line %level: - %message %newline" />
+ </layout>
+ </appender>
+
+ <!-- Set root logger level to DEBUG-->
+ <root>
+ <level value="DEBUG" />
+ <appender-ref ref="Console" />
+ <appender-ref ref="RollingFileDebug" />
+ <appender-ref ref="RollingFileDebug2" />
+ </root>
+</log4net>
View
73 GSTEvaluation/export/ComparisonExcelExport.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
@@ -21,22 +22,23 @@ protected override void WriteData(WorkBook wb, EvaluationRunModel model)
protected override void WriteData(WorkBook wb, IList<ComparisonHistoryModel> data)
{
- /*
- if(1 < data.Count)
- wb.insertSheets(0, data.Count - 1);
- /* */
-
+ wb.insertSheets(1, data.Count);
wb.setSheetName(0, "data");
for (int i = 0; i < data.Count; i++)
{
- //Console.WriteLine("COUNT:{0},{1}", list.Count, i);
+ wb.Sheet = DataSheetIndex;
var compHistory = data[i];
- var columnOffset = i*2;
+ var columnOffset = i*3;
+
+ if (!compHistory.Data.Any())
+ {
+ Console.WriteLine("no data points for: {0}", compHistory.Name);
+ continue;
+ }
+ Console.WriteLine("Setting column headers: offset: {0}, name: {1}", columnOffset, compHistory.Name);
- wb.setEntry(0, columnOffset, "EvalRun ID");
- wb.setEntry(0, columnOffset + 1, string.Format("Result {0}", compHistory.Name));
WriteDataColumns(wb, compHistory, columnOffset);
WriteChart(wb, compHistory, columnOffset, i + 1);
}
@@ -53,35 +55,36 @@ private void WriteChart(WorkBook wb, ComparisonHistoryModel compHistory, int col
{
Console.WriteLine("write chart sheetIndex: {0}", sheetIndex);
- //insert sheet and transform into chart sheeet
- wb.insertSheets(sheetIndex, 1);
- ChartShape tChart = wb.addChartSheet(sheetIndex);
+ // transform sheet into chart sheeet
+
+ var chartShape = wb.addChartSheet(sheetIndex);
wb.Sheet = sheetIndex;
wb.PrintScaleFitToPage = true;
wb.PrintLandscape = true;
- wb.setSheetName(sheetIndex, string.Format("Diagramm_{0}", compHistory.Name));
- tChart.ChartType = ChartShape.Scatter;
+ wb.setSheetName(sheetIndex, string.Format("Diagram_{0}", compHistory.Name));
+ chartShape.ChartType = ChartShape.Scatter;
- tChart.setAxisTitle(ChartShape.XAxis, 0, "Evaluation Run ID");
- tChart.setAxisTitle(ChartShape.YAxis, 0, "Similarity [%]");
+ chartShape.setAxisTitle(ChartShape.XAxis, 0, "Evaluation Run ID");
+ chartShape.setAxisTitle(ChartShape.YAxis, 0, "Similarity [%]");
- ChartFormat tFormat = tChart.PlotFormat;
+ ChartFormat tFormat = chartShape.PlotFormat;
tFormat.setLineNone();
- tChart.setSeriesName(0, compHistory.Name);
- var format = tChart.getSeriesFormat(0);
+ chartShape.setSeriesName(0, compHistory.Name);
+ var format = chartShape.getSeriesFormat(0);
format.MarkerStyle = ChartFormat.MarkerCircle;
- tChart.setSeriesFormat(0, format);
-
- string xFormula = string.Format("data!${0}${1}:${0}${2}", getDataColumnIndex(columnOffset), 2, compHistory.Data.Count());
- Console.WriteLine("Hello: {0}", xFormula);
- tChart.setSeriesXValueFormula(0, xFormula);
- string yFormula = string.Format("data!${0}${1}:${0}${2}", getDataColumnIndex(columnOffset + 1),2, compHistory.Data.Count());
- tChart.setSeriesYValueFormula(0, yFormula);
+ chartShape.setSeriesFormat(0, format);
+ string xFormula = string.Format("data!${0}${1}:${0}${2}", GetDataColumnIndex(columnOffset), 2, compHistory.Data.Count());
+
+ chartShape.setSeriesXValueFormula(0, xFormula);
+ string yFormula = string.Format("data!${0}${1}:${0}${2}", GetDataColumnIndex(columnOffset + 2),2, compHistory.Data.Count());
+ Console.WriteLine("xFormula: {0}, yFormula: {1}", xFormula, yFormula);
+
+ chartShape.setSeriesYValueFormula(0, yFormula);
}
/// <summary>
@@ -93,18 +96,20 @@ private void WriteChart(WorkBook wb, ComparisonHistoryModel compHistory, int col
private void WriteDataColumns(WorkBook wb, ComparisonHistoryModel compHistory, int columnOffset)
{
var list = compHistory.Data.ToList();
+
+ Console.WriteLine("Data column: {0}, {1}, count: {2}", compHistory.Name, columnOffset, list.Count);
+
+ wb.setEntry(0, columnOffset, "EvalRun ID");
+ wb.setEntry(0, columnOffset + 1, "Label");
+ wb.setEntry(0, columnOffset + 2, string.Format("Result {0}", compHistory.Name));
+
for(int i = 0; i < list.Count; i++)
{
var tuple = list[i];
- wb.setEntry(i + 1, columnOffset, tuple.Item1.ToString()); // eval Run ID
- wb.setEntry(i + 1, columnOffset + 1, tuple.Item2.ToString()); // result
+ wb.setEntry(i + 1, columnOffset, tuple.EvaluationRunID.ToString(CultureInfo.InvariantCulture));
+ wb.setEntry(i + 1, columnOffset + 1, tuple.EvaluationRunLabel);
+ wb.setEntry(i + 1, columnOffset + 2, tuple.Result.ToString(CultureInfo.InvariantCulture));
}
}
-
-
- private static char getDataColumnIndex(int aMeasurementIndex)
- {
- return (char)('A' + (char)aMeasurementIndex);
- }
}
}
View
159 GSTEvaluation/export/CompleteComparisonReport.cs
@@ -0,0 +1,159 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using GSTAppLogic.ext;
+using GSTEvaluation.model;
+using GSTLibrary.tile;
+using GSTLibrary.token;
+using SmartXLS;
+using System.IO;
+using Tokenizer;
+using CTokenizer;
+using System.Globalization;
+
+namespace GSTEvaluation.export
+{
+ /// <summary>
+ /// runs MUTEX for a subset of all available
+ /// </summary>
+ class CompleteComparisonReport : FileExport
+ {
+ private readonly string testName;
+ public CompleteComparisonReport(string name)
+ {
+ testName = name;
+ Name = "CompleteComparison\\" + name;
+ }
+
+ protected override string FileExtension
+ {
+ get { return ".txt"; }
+ }
+
+ protected override MemoryStream CreateContent(EvaluationRunModel model)
+ {
+ throw new NotImplementedException();
+ }
+
+ protected override MemoryStream CreateContent(IList<ComparisonHistoryModel> data)
+ {
+ throw new NotImplementedException();
+ }
+
+ public void Run()
+ {
+ var cartesianProduct = GetUniqueCartesianProduct(testName);
+ var results =
+ cartesianProduct.Select(elem => Calculate(testName, elem)).Select(
+ el => string.Format("{0}: {1}", el[0], el[1]));
+
+ File.WriteAllLines(FilePath, results);
+ }
+
+ /// <summary>
+ /// evaluation of GST speed only.
+ /// DO NOT TRY THIS AT HOME!
+ /// </summary>
+ public static void PerformTest(string testName)
+ {
+ //File.Copy(@"D:\test\MUTEX\CompleteComparison\template-complete.xlsx", @"D:\test\MUTEX\CompleteComparison\template-complete2.xlsx", true);
+
+ var cartesianProduct = GetUniqueCartesianProduct(testName);
+ var results = cartesianProduct.Select(elem => Calculate(testName, elem));
+
+
+
+ var wb = new WorkBook();
+ wb.readXLSX(@"D:\test\MUTEX\CompleteComparison\template-complete2.xlsx");
+
+ SetActiveSheet(testName, wb);
+
+ int i = 0;
+ try
+ {
+ foreach (var elem in results)
+ {
+ wb.setEntry(i + 1, 1, string.Format("{0}.0", elem[1]));
+
+ i++;
+ }
+ }
+ catch (NullReferenceException ex)
+ {
+ Console.WriteLine(ex.ToString());
+ Console.ReadLine();
+ Environment.Exit(0);
+ }
+
+
+ wb.writeXLSX(@"D:\test\MUTEX\CompleteComparison\heyja.xlsx");
+ }
+
+ private static List<int[]> GetUniqueCartesianProduct(string testName)
+ {
+ var files = Directory.GetFiles(string.Format(@"D:\test\MUTEX\CompleteComparison\sources\{0}", testName),
+ @"main.c", SearchOption.AllDirectories).Select(p =>
+ {
+ var dn =
+ new FileInfo(p).
+ DirectoryName;
+ return
+ Int32.Parse(
+ dn.Substring(
+ dn.LastIndexOf(
+ '\\') + 1));
+ }).ToList();
+
+ var cp = from first in files
+ from second in files
+ orderby first , second
+ select new[] {first, second};
+
+ var cartesianProduct = cp.Where(elem => elem[0] < elem[1]).ToList();
+ return cartesianProduct;
+ }
+
+ private static void SetActiveSheet(string testName, WorkBook wb)
+ {
+ for (int i = 0; i < wb.NumSheets; i++)
+ {
+ if (testName == wb.getSheetName(i))
+ {
+ Console.WriteLine("Setting active sheet {0}", i);
+ wb.Sheet = i;
+ break;
+ }
+ }
+ }
+
+ private static string[] Calculate(string testName, int[] elem)
+ {
+ var path1 = string.Format(
+ @"D:\test\MUTEX\CompleteComparison\sources\{0}\{1:00}\main.c",
+ testName, elem[0]);
+ var path2 = string.Format(
+ @"D:\test\MUTEX\CompleteComparison\sources\{0}\{1:00}\main.c",
+ testName, elem[1]);
+ return new[]
+ {
+ string.Format("{0}-{1}", elem[0], elem[1]),
+ AppHelper.CompareFiles(path1, path2).ToString(CultureInfo.InvariantCulture)
+ };
+ }
+
+ private static GSTTokenList<GSTToken<TokenWrapper>> GetTokens(FileInfo file)
+ {
+ string source = File.ReadAllText(file.FullName);
+ var tokens = LexerHelper.CreateLexerFromSource(source).GetTokenWrappers().ToList();
+
+ return tokens.ToGSTTokenList();
+ }
+
+ private static GSTTokenList<GSTToken<TokenWrapper>> GetTokens(string file)
+ {
+ return GetTokens(new FileInfo(file));
+ }
+ }
+}
View
18 GSTEvaluation/export/ExcelExport.cs
@@ -13,6 +13,14 @@ namespace GSTEvaluation.export
/// </summary>
abstract class ExcelExport : FileExport
{
+ /// <summary>
+ /// this is the index of the sheet which contains all of the data
+ /// </summary>
+ protected const int DataSheetIndex = 0;
+
+ /// <summary>
+ /// the file extension is .xlsx
+ /// </summary>
protected override string FileExtension
{
get { return ".xlsx"; }
@@ -30,7 +38,7 @@ protected ExcelExport(string name)
/// <returns></returns>
protected override MemoryStream CreateContent(EvaluationRunModel model)
{
- WorkBook wb = new WorkBook();
+ var wb = new WorkBook();
WriteData(wb, model);
var stream = new MemoryStream();
@@ -41,7 +49,7 @@ protected override MemoryStream CreateContent(EvaluationRunModel model)
protected override MemoryStream CreateContent(IList<ComparisonHistoryModel> data)
{
- WorkBook wb = new WorkBook();
+ var wb = new WorkBook();
WriteData(wb, data);
var stream = new MemoryStream();
@@ -56,5 +64,11 @@ protected override MemoryStream CreateContent(IList<ComparisonHistoryModel> data
protected abstract void WriteData(WorkBook wb, EvaluationRunModel model);
protected abstract void WriteData(WorkBook wb, IList<ComparisonHistoryModel> data);
+
+
+ protected static char GetDataColumnIndex(int aDataColumn)
+ {
+ return (char)('A' + (char)aDataColumn);
+ }
}
}
View
2 GSTEvaluation/export/FileExport.cs
@@ -13,7 +13,7 @@ namespace GSTEvaluation.export
/// </summary>
abstract class FileExport : IExport
{
- public static readonly string RESULT_DIRECTORY = @"D:\test\Results";
+ public static readonly string RESULT_DIRECTORY = @"D:\test\MUTEX\Results";
private string filePath;
View
6 GSTEvaluation/export/ListResultsExport.cs
@@ -3,6 +3,7 @@
using System.Linq;
using System.Text;
using GSTEvaluation.model;
+using log4net;
namespace GSTEvaluation.export
{
@@ -11,6 +12,7 @@ namespace GSTEvaluation.export
/// </summary>
class ListResultsExport : IExport
{
+ private static readonly ILog cLogger = LogManager.GetLogger(typeof(ListResultsExport));
public string Name
{
get { return string.Empty; }
@@ -19,10 +21,10 @@ public string Name
public void Run(EvaluationRunModel model)
{
- Console.WriteLine("Evaluation Run {0}", model.ID);
+ cLogger.DebugFormat("Evaluation Run {0}", model.ID);
foreach (var comparison in model.Comparisons)
{
- Console.WriteLine("{0}:\t\t{1}", comparison.Name, comparison.Result);
+ cLogger.InfoFormat("{0}:\t\t{1}", comparison.Name, comparison.Result);
}
}
View
116 GSTEvaluation/export/RuntimeExcelExport.cs
@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using GSTEvaluation.model;
+using SmartXLS;
+using log4net;
+
+namespace GSTEvaluation.export
+{
+ class RuntimeExcelExport : ExcelExport
+ {
+ private static readonly ILog cLogger = LogManager.GetLogger(typeof(RuntimeExcelExport));
+ public RuntimeExcelExport(string name) : base(name)
+ {
+ }
+
+ protected override void WriteData(WorkBook wb, EvaluationRunModel model)
+ {
+ throw new NotImplementedException("not supported for this export, please use multi method");
+ }
+
+ protected override void WriteData(WorkBook wb, IList<ComparisonHistoryModel> data)
+ {
+ wb.insertSheets(1, data.Count);
+ wb.setSheetName(0, "data");
+
+ for (int i = 0; i < data.Count; i++)
+ {
+ wb.Sheet = DataSheetIndex;
+ var compHistory = data[i];
+ var columnOffset = i*3;
+
+ if (!compHistory.Data.Any())
+ {
+ cLogger.DebugFormat("no data points for: {0}", compHistory.Name);
+ continue;
+ }
+
+ cLogger.DebugFormat("Setting column headers: offset: {0}, name: {1}", columnOffset, compHistory.Name);
+
+ WriteDataColumns(wb, compHistory, columnOffset);
+ WriteChart(wb, compHistory, columnOffset, i + 1);
+ }
+ /* */
+ }
+
+ /// <summary>
+ /// writes the chart
+ /// </summary>
+ /// <param name="wb"></param>
+ /// <param name="compHistory"></param>
+ /// <param name="columnOffset"></param>
+ private void WriteChart(WorkBook wb, ComparisonHistoryModel compHistory, int columnOffset, int sheetIndex)
+ {
+ cLogger.DebugFormat("write chart sheetIndex: {0}", sheetIndex);
+
+ // transform sheet into chart sheeet
+
+ ChartShape chartShape = wb.addChartSheet(sheetIndex);
+
+ wb.Sheet = sheetIndex;
+
+ wb.PrintScaleFitToPage = true;
+ wb.PrintLandscape = true;
+
+ wb.setSheetName(sheetIndex, string.Format("Diagram_{0}", compHistory.Name));
+ chartShape.ChartType = ChartShape.Scatter;
+
+ chartShape.setAxisTitle(ChartShape.XAxis, 0, "Evaluation Run ID");
+ chartShape.setAxisTitle(ChartShape.YAxis, 0, "Runtime [ms]");
+
+ ChartFormat tFormat = chartShape.PlotFormat;
+ tFormat.setLineNone();
+
+ chartShape.setSeriesName(0, compHistory.Name);
+ var format = chartShape.getSeriesFormat(0);
+ format.MarkerStyle = ChartFormat.MarkerCircle;
+ chartShape.setSeriesFormat(0, format);
+
+ string xFormula = string.Format("data!${0}${1}:${0}${2}", GetDataColumnIndex(columnOffset), 2, compHistory.Data.Count());
+
+ chartShape.setSeriesXValueFormula(0, xFormula);
+ string yFormula = string.Format("data!${0}${1}:${0}${2}", GetDataColumnIndex(columnOffset + 2),2, compHistory.Data.Count());
+ cLogger.DebugFormat("xFormula: {0}, yFormula: {1}", xFormula, yFormula);
+
+ chartShape.setSeriesYValueFormula(0, yFormula);
+ }
+
+ /// <summary>
+ /// write
+ /// </summary>
+ /// <param name="wb"></param>
+ /// <param name="compHistory"></param>
+ /// <param name="columnOffset"></param>
+ private void WriteDataColumns(WorkBook wb, ComparisonHistoryModel compHistory, int columnOffset)
+ {
+ var list = compHistory.Data.ToList();
+
+ Console.WriteLine("Data column: {0}, {1}, count: {2}", compHistory.Name, columnOffset, list.Count);
+
+ wb.setEntry(0, columnOffset, "EvalRun ID");
+ wb.setEntry(0, columnOffset + 1, "Label");
+ wb.setEntry(0, columnOffset + 2, string.Format("Runtime {0}", compHistory.Name));
+
+ for(int i = 0; i < list.Count; i++)
+ {
+ var tuple = list[i];
+ wb.setEntry(i + 1, columnOffset, tuple.EvaluationRunID.ToString(CultureInfo.InvariantCulture));
+ wb.setEntry(i + 1, columnOffset + 1, tuple.EvaluationRunLabel);
+ wb.setEntry(i + 1, columnOffset + 2, tuple.Runtime.ToString(CultureInfo.InvariantCulture));
+ }
+ }
+ }
+}
View
22 GSTEvaluation/model/ComparisonHistoryModel.cs
@@ -8,18 +8,38 @@ namespace GSTEvaluation.model
{
class ComparisonHistoryModel
{
+ internal struct HistoryDataPoint
+ {
+ public Int64 EvaluationRunID;
+ public string EvaluationRunLabel;
+ public Int32 Result;
+ public Int64 Runtime;
+ }
+
public string Name { get; private set; }
/// <summary>
/// returns the data in tuples of [EvaluationRunID, Comparison-Result]
/// </summary>
- public IEnumerable<Tuple<Int64, Int32>> Data { get; private set; }
+ public IEnumerable<HistoryDataPoint> Data { get; private set; }
public ComparisonHistoryModel(string comparisonName)
{
Name = comparisonName;
Data = SQLFacade.Instance.GetComparisonHistory(comparisonName);
}
+
+ public static IList<ComparisonHistoryModel> AllHistories
+ {
+ get
+ {
+ var list = new List<ComparisonHistoryModel>();
+
+ foreach(string name in SQLFacade.Instance.GetComparisonNames())
+ list.Add(new ComparisonHistoryModel(name));
+ return list;
+ }
+ }
}
}
View
24 GSTEvaluation/model/ComparisonModel.cs
@@ -1,14 +1,17 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using CTokenizer;
using GSTAppLogic.ext;
+using GSTAppLogic.templating;
using GSTEvaluation.storage;
using GSTLibrary.tile;
using GSTLibrary.token;
using Tokenizer;
+using log4net;
namespace GSTEvaluation.model
{
@@ -17,6 +20,7 @@ namespace GSTEvaluation.model
/// </summary>
class ComparisonModel
{
+ private static readonly ILog cLogger = LogManager.GetLogger(typeof(ComparisonModel));
public Int64 EvaluationRunID { get; private set; }
public string Name { get; private set; }
@@ -33,15 +37,25 @@ class ComparisonModel
/// <summary>
/// takes the two sources and performs the GST
/// </summary>
- /// <param name="source1"></param>
- /// <param name="source2"></param>
+ /// <param name="sourcePath1"></param>
+ /// <param name="sourcePath2"></param>
public ComparisonModel(string name, Int64 evalRunID, string sourcePath1, string sourcePath2)
{
Name = name;
EvaluationRunID = evalRunID;
+
+ var directory = Path.GetDirectoryName(Path.GetFullPath(sourcePath1));
+ bool tmplFileExists = File.Exists(Path.Combine(directory, "template.c"));
- var tokens1 = LexerHelper.CreateLexer(sourcePath1).GetTokenWrappers();
- var tokens2 = LexerHelper.CreateLexer(sourcePath2).GetTokenWrappers();
+ var watch = Stopwatch.StartNew();
+
+ var tmplFile = Directory.GetFiles(directory, "template.c").FirstOrDefault();
+ var path1 = tmplFileExists ? TemplatingHelper.StripTemplateFromSourceFile(sourcePath1, tmplFile) : sourcePath1;
+ var path2 = tmplFileExists ? TemplatingHelper.StripTemplateFromSourceFile(sourcePath2, tmplFile) : sourcePath2;
+ var tokens1 = LexerHelper.CreateLexer(path1).GetTokenWrappers();
+ var tokens2 = LexerHelper.CreateLexer(path2).GetTokenWrappers();
+
+ cLogger.DebugFormat("TokenStream Length: {0} -- {1}", tokens1.Count(), tokens2.Count());
var algo = new GSTAlgorithm<GSTToken<TokenWrapper>>(
tokens1.ToGSTTokenList<TokenWrapper>(),
tokens2.ToGSTTokenList<TokenWrapper>());
@@ -51,7 +65,7 @@ public ComparisonModel(string name, Int64 evalRunID, string sourcePath1, string
Source1 = new SourceModel(Path.GetFileNameWithoutExtension(sourcePath1), tokens1.GetJoinedTokenString());
Source2 = new SourceModel(Path.GetFileNameWithoutExtension(sourcePath2), tokens2.GetJoinedTokenString());
- SQLFacade.Instance.CreateComparison(name, Result, evalRunID, Source1.ID, Source2.ID);
+ SQLFacade.Instance.CreateComparison(name, Result, watch.ElapsedMilliseconds, evalRunID, Source1.ID, Source2.ID);
}
}
}
View
12 GSTEvaluation/model/EvaluationRunModel.cs
@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using GSTEvaluation.storage;
+using log4net;
namespace GSTEvaluation.model
{
@@ -13,6 +15,7 @@ namespace GSTEvaluation.model
/// </summary>
class EvaluationRunModel
{
+ private static readonly ILog cLogger = LogManager.GetLogger(typeof(EvaluationRunModel));
public Int64 ID { get; private set; }
public IEnumerable<ComparisonModel> Comparisons { get; private set; }
@@ -30,15 +33,16 @@ public EvaluationRunModel(string directory)
{
try
{
- var file1 = dir.GetFiles()[0];
- var file2 = dir.GetFiles()[1];
+ var file1 = dir.GetFiles("main*")[0];
+ var file2 = dir.GetFiles("main*")[1];
+ var watch = Stopwatch.StartNew();
var comparison = new ComparisonModel(dir.Name, ID, file1.FullName, file2.FullName);
+ cLogger.DebugFormat("comparison {0} took {1}", comparison.Name, watch.ElapsedMilliseconds);
list.Add(comparison);
}
catch (Exception ex)
{
- Console.WriteLine("could not make comparison: {0}", dir.FullName);
- Console.WriteLine(ex.ToString());
+ cLogger.Debug(string.Format("could not make comparison: {0}", dir.FullName), ex);
}
}
View
51 GSTEvaluation/storage/SQLFacade.cs
@@ -4,6 +4,7 @@
using System.IO;
using System.Linq;
using System.Text;
+using GSTEvaluation.model;
using log4net;
namespace GSTEvaluation.storage
@@ -62,9 +63,10 @@ private void CreateTables()
"CREATE TABLE IF NOT EXISTS " + COMPARISON_TABLE_NAME + " " +
"(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, " +
"source_1_id INTEGER NOT NULL, source_2_id INTEGER NOT NULL, " +
- "run_id INTEGER NOT NULL, result INTEGER NOT NULL)",
+ "run_id INTEGER NOT NULL, result INTEGER NOT NULL, runtime INTEGER NOT NULL)",
"CREATE TABLE IF NOT EXISTS " + EVAL_RUN_TABLE_NAME + " " +
"(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " +
+ "label TEXT, " +
"datetime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)"
};
@@ -113,12 +115,12 @@ public Int64 InsertSource(string joinedTokenStream, string name)
return (Int64)command.ExecuteScalar();
}
- public Int64 CreateComparison(string name, Int32 result, Int64 evalRunId, Int64 source1ID, Int64 source2ID)
+ public Int64 CreateComparison(string name, Int32 result, Int64 runtime, Int64 evalRunId, Int64 source1ID, Int64 source2ID)
{
var command = new SQLiteCommand(connection)
{
- CommandText = string.Format("INSERT INTO {0}(name, run_id, source_1_id, source_2_id, result) " +
- "VALUES('{1}', {2}, {3}, {4}, {5})", COMPARISON_TABLE_NAME, name, evalRunId, source1ID, source2ID, result)
+ CommandText = string.Format("INSERT INTO {0}(name, run_id, source_1_id, source_2_id, result, runtime) " +
+ "VALUES('{1}', {2}, {3}, {4}, {5}, {6})", COMPARISON_TABLE_NAME, name, evalRunId, source1ID, source2ID, result, runtime)
};
command.ExecuteNonQuery();
@@ -133,27 +135,56 @@ public Int64 CreateComparison(string name, Int32 result, Int64 evalRunId, Int64
/// </summary>
/// <param name="comparisonName"></param>
/// <returns></returns>
- public IEnumerable<Tuple<Int64, Int32>> GetComparisonHistory(string comparisonName)
+ public IEnumerable<ComparisonHistoryModel.HistoryDataPoint> GetComparisonHistory(string comparisonName)
{
var command = new SQLiteCommand(connection)
{
- CommandText = string.Format("SELECT eval.id, comp.result " +
+ CommandText = string.Format("SELECT eval.id, eval.label, comp.result, comp.runtime " +
"FROM {0} AS comp INNER JOIN {1} AS eval ON comp.run_id = eval.id " +
"WHERE comp.name='{2}' " +
"ORDER BY eval.datetime ASC ", COMPARISON_TABLE_NAME, EVAL_RUN_TABLE_NAME, comparisonName)
};
var reader = command.ExecuteReader();
- var list = new List<Tuple<Int64, Int32>>();
+ var list = new List<ComparisonHistoryModel.HistoryDataPoint>();
while(reader.Read())
{
- list.Add(new Tuple<Int64, Int32>(
- reader.GetInt64(reader.GetOrdinal("id")),
- reader.GetInt32(reader.GetOrdinal("result"))));
+ var label = reader.GetValue(reader.GetOrdinal("label"));
+ list.Add(new ComparisonHistoryModel.HistoryDataPoint
+ {
+ EvaluationRunID = reader.GetInt64(reader.GetOrdinal("id")),
+ EvaluationRunLabel = label.ToString(),
+ Result = reader.GetInt32(reader.GetOrdinal("result")),
+ Runtime = reader.GetInt64(reader.GetOrdinal("runtime"))
+ });
}
+ return list;
+ }
+
+ /// <summary>
+ /// returns all the names of the different comparisons stored in the DB
+ /// </summary>
+ /// <returns></returns>
+ internal IEnumerable<string> GetComparisonNames()
+ {
+ var command = new SQLiteCommand(connection)
+ {
+ CommandText = string.Format("SELECT DISTINCT name " +
+ "FROM {0} AS comp " +
+ "ORDER BY comp.name", COMPARISON_TABLE_NAME)
+ };
+
+ var reader = command.ExecuteReader();
+
+ var list = new List<string>();
+
+ while (reader.Read())
+ {
+ list.Add(reader.GetString(reader.GetOrdinal("name")));
+ }
return list;
}
View
BIN GSTEvaluation120807(1).vsp
Binary file not shown.
View
BIN GSTEvaluation120807.vsp
Binary file not shown.
View
BIN GSTEvaluation120809.vsp
Binary file not shown.
View
1 GSTLibrary/test/tile/GSTAlgorithmTest.cs
@@ -62,7 +62,6 @@ public void TestIsInRange()
Assert.False(GSTAlgorithm<GSTToken>.IsInRange(5, 6, 7), "below range");
Assert.True(GSTAlgorithm<GSTToken>.IsInRange(5, 3, 5), "upper edge");
Assert.True(GSTAlgorithm<GSTToken>.IsInRange(5, 5, 7), "lower edge");
-
}
[Test]
View
10 GSTLibrary/tile/GSTAlgorithm.cs
@@ -3,6 +3,7 @@
using System.Linq;
using GSTLibrary.exception;
using GSTLibrary.token;
+using System.Diagnostics;
namespace GSTLibrary.tile
{
@@ -12,6 +13,8 @@ namespace GSTLibrary.tile
/// <typeparam name="T"></typeparam>
public class GSTAlgorithm<T> where T : GSTToken
{
+ //private long counter;
+
/// <summary>
/// the default minimum match length used
/// </summary>
@@ -127,6 +130,8 @@ public void DoOneRun()
if(Finished)
throw new GSTException("algorithm is finished");
+ var watch = Stopwatch.StartNew();
+
var listMatches = new List<Tile<T>>();
LastMaximumMatch = MML;
@@ -140,6 +145,7 @@ public void DoOneRun()
// for every token in B that is unmarked and matches tokA
foreach(var tB in ListB.Where(t => !t.Marked).Where(tokA.EqualsTokenValue))
{
+ //counter++;
var tokB = tB; // CLOSURE
int indB = ListB.IndexOf(tokB);
@@ -159,6 +165,7 @@ public void DoOneRun()
}
TilesMatchedInLastRun = listMatches;
+ //Console.WriteLine("one run({1}) took {0} ms", watch.ElapsedMilliseconds, counter);
}
/// <summary>
@@ -269,7 +276,7 @@ private void MarkTileAsMatched(Tile<T> tile)
/// <returns></returns>
private Tile<T> CreateTile(int indA, int indB, int matchLength)
{
- List<T> list = new List<T>();
+ var list = new List<T>();
for (int i = indA; i < indA + matchLength; i++)
{
list.Add(ListA[i]);
@@ -295,6 +302,7 @@ private int CalculateMatchLength(int offsetA, int offsetB)
var cListA = ListA.Count;
var cListB = ListB.Count;
+
do
{
var tokA = ListA[offsetA + index];
View
1 GSTLibrary/token/GSTToken.cs
@@ -59,6 +59,7 @@ public override string ToString()
public override bool EqualsTokenValue(GSTToken other)
{
+
if(other is GSTToken<T>)
{
return ((GSTToken<T>) other).Token.Equals(Token);
View
15 GSTTests.VisualState.xml
@@ -16,12 +16,13 @@
<Node UniqueName="[0-1026]D:\wissen\csharp\GreedyStringTiling\CTokenizer\bin\Debug\CTokenizer.dll" Expanded="true" />
<Node UniqueName="[0-1027]CTokenizer" Expanded="true" />
<Node UniqueName="[0-1028]CTokenizer.test" Expanded="true" />
- <Node UniqueName="[0-1031]D:\wissen\csharp\GreedyStringTiling\GSTAppLogic\bin\Debug\GSTAppLogic.dll" Expanded="true" />
- <Node UniqueName="[0-1032]GSTAppLogic" Expanded="true" />
- <Node UniqueName="[0-1033]GSTAppLogic.test" Expanded="true" />
- <Node UniqueName="[0-1034]GSTAppLogic.test.model" Expanded="true" />
- <Node UniqueName="[0-1038]D:\wissen\csharp\GreedyStringTiling\GSTConsole\bin\Debug\GSTConsole.exe" Expanded="true" />
- <Node UniqueName="[0-1039]GSTConsole" Expanded="true" />
- <Node UniqueName="[0-1040]GSTConsole.test" Expanded="true" />
+ <Node UniqueName="[0-1033]D:\wissen\csharp\GreedyStringTiling\GSTAppLogic\bin\Debug\GSTAppLogic.dll" Expanded="true" />
+ <Node UniqueName="[0-1034]GSTAppLogic" Expanded="true" />
+ <Node UniqueName="[0-1035]GSTAppLogic.test" Expanded="true" />
+ <Node UniqueName="[0-1036]GSTAppLogic.test.model" Expanded="true" />
+ <Node UniqueName="[0-1037]GSTAppLogic.test.templating" Expanded="true" />
+ <Node UniqueName="[0-1041]D:\wissen\csharp\GreedyStringTiling\GSTConsole\bin\Debug\GSTConsole.exe" Expanded="true" />
+ <Node UniqueName="[0-1042]GSTConsole" Expanded="true" />
+ <Node UniqueName="[0-1043]GSTConsole.test" Expanded="true" />
</Nodes>
</VisualState>
View
5 GreedyStringTiling.sln
@@ -17,6 +17,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GreedyStringTiling", "Greed
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GSTEvaluation", "GSTEvaluation\GSTEvaluation.csproj", "{1F213970-9937-4559-AD48-B0504D449E14}"
EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{30B69E9C-849F-42F5-AF66-68BF39634F0F}"
+ ProjectSection(SolutionItems) = preProject
+ Performance1.psess = Performance1.psess
+ EndProjectSection
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
View
25 Performance1.psess
@@ -0,0 +1,25 @@
+<VSPerformanceSession Version="1.00">
+ <Options>
+ <CollectionMethod>Sampling</CollectionMethod>
+ <AllocationMethod>None</AllocationMethod>
+ <AddReport>true</AddReport>
+ <UniqueReport>Timestamp</UniqueReport>
+ <SamplingMethod>Cycles</SamplingMethod>
+ <CycleCount>10000000</CycleCount>
+ <PageFaultCount>10</PageFaultCount>
+ <SysCallCount>10</SysCallCount>
+ <SamplingCounter PlatformID="00000000" CounterID="0000000000000000" ReloadValue="000000000000000a" />
+ <RelocateBinaries>false</RelocateBinaries>
+ <CollectOnHpcCluster>false</CollectOnHpcCluster>
+ <HPCSettings>
+ <TargetSelection>UnknownDisabled</TargetSelection>
+ </HPCSettings>
+ <HardwareCounters EnableHWCounters="false" />
+ </Options>
+ <PreinstrumentEvent>
+ <InstrEventExclude>false</InstrEventExclude>
+ </PreinstrumentEvent>
+ <PostinstrumentEvent>
+ <InstrEventExclude>false</InstrEventExclude>
+ </PostinstrumentEvent>
+</VSPerformanceSession>
View
1 Tokenizer/TokenType.cs
@@ -12,7 +12,6 @@ public class TokenType
/// </summary>
public Int32 Type { get; set; }
-
public override bool Equals(object obj)
{
if (obj == null || GetType() != obj.GetType())
View
10 Tokenizer/TokenWrapper.cs
@@ -10,6 +10,11 @@ namespace Tokenizer
public class TokenWrapper
{
/// <summary>
+ /// stores the hash code of the underlying token to speed up comparison
+ /// </summary>
+ private readonly long tokenHashCode;
+
+ /// <summary>
/// the saved token object from ANTLR lexers
/// </summary>
private IToken Token { get; set; }
@@ -33,6 +38,7 @@ public TokenWrapper(IToken token, string tokenName)
Token = token;
Text = token.Text;
Type = new TokenType {Type = token.Type};
+ tokenHashCode = Type.GetHashCode();
}
/// <summary>
@@ -56,7 +62,9 @@ public override bool Equals(object obj)
null == other.Token)
return false;
- return other.Type.Equals(Type);
+ return tokenHashCode == other.tokenHashCode;
+
+ //return other.Type.Equals(Type);
}
public override int GetHashCode()

0 comments on commit f1a3e6d

Please sign in to comment.
Something went wrong with that request. Please try again.