Skip to content

Commit

Permalink
-add: Tokenizer project (DLL), contains base classes for tokenizers
Browse files Browse the repository at this point in the history
-add: TokenType to hide implementation class of the token type
-refactor: Hiding implementation types for TokenWrapper
  • Loading branch information
yas4891 committed May 4, 2012
1 parent f35f977 commit 65c622f
Show file tree
Hide file tree
Showing 10 changed files with 173 additions and 13 deletions.
8 changes: 6 additions & 2 deletions CTokenizer/CTokenizer.csproj
Expand Up @@ -54,15 +54,19 @@
<Compile Include="CLexer.cs" />
<Compile Include="MutexTokenImpl.cs" />
<Compile Include="test\LexerHelperTest.cs" />
<Compile Include="TokenWrapper.cs" />
<Compile Include="LexerHelper.cs" />
<Compile Include="MutexCLexer.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="MutexCLexer.g" />
</ItemGroup>
<ItemGroup />
<ItemGroup>
<ProjectReference Include="..\Tokenizer\Tokenizer.csproj">
<Project>{044DABC4-13F0-4581-A9AF-FF645FD460EB}</Project>
<Name>Tokenizer</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
Expand Down
1 change: 1 addition & 0 deletions CTokenizer/LexerHelper.cs
Expand Up @@ -4,6 +4,7 @@
using System.Reflection;
using System.Text;
using Antlr.Runtime;
using Tokenizer;

namespace CTokenizer
{
Expand Down
4 changes: 4 additions & 0 deletions GSTAppLogic/GSTAppLogic.csproj
Expand Up @@ -71,6 +71,10 @@
<Project>{B80B2459-2992-4F3A-A6C4-EE692EDA01FF}</Project>
<Name>GSTLibrary</Name>
</ProjectReference>
<ProjectReference Include="..\Tokenizer\Tokenizer.csproj">
<Project>{044DABC4-13F0-4581-A9AF-FF645FD460EB}</Project>
<Name>Tokenizer</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Expand Down
1 change: 1 addition & 0 deletions GSTAppLogic/app/model/ComparisonModel.cs
Expand Up @@ -7,6 +7,7 @@
using GSTAppLogic.ext;
using GSTLibrary.tile;
using GSTLibrary.token;
using Tokenizer;

namespace GSTAppLogic.app.model
{
Expand Down
6 changes: 2 additions & 4 deletions GSTAppLogic/ext/LexerExtensions.cs
@@ -1,9 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using CTokenizer;
using GSTLibrary.token;
using Tokenizer;

namespace GSTAppLogic.ext
{
Expand Down
12 changes: 12 additions & 0 deletions GreedyStringTiling.sln
Expand Up @@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataRepository", "DataRepos
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GSTAppLogic", "GSTAppLogic\GSTAppLogic.csproj", "{D00B37DB-1567-4F8C-9476-A31A8005D7D2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tokenizer", "Tokenizer\Tokenizer.csproj", "{044DABC4-13F0-4581-A9AF-FF645FD460EB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -71,6 +73,16 @@ Global
{D00B37DB-1567-4F8C-9476-A31A8005D7D2}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{D00B37DB-1567-4F8C-9476-A31A8005D7D2}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{D00B37DB-1567-4F8C-9476-A31A8005D7D2}.Release|x86.ActiveCfg = Release|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Debug|x86.ActiveCfg = Debug|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Release|Any CPU.Build.0 = Release|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{044DABC4-13F0-4581-A9AF-FF645FD460EB}.Release|x86.ActiveCfg = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
36 changes: 36 additions & 0 deletions Tokenizer/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// Allgemeine Informationen über eine Assembly werden über die folgenden
// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern,
// die mit einer Assembly verknüpft sind.
[assembly: AssemblyTitle("Tokenizer")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Tokenizer")]
[assembly: AssemblyCopyright("Copyright © 2012")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

// Durch Festlegen von ComVisible auf "false" werden die Typen in dieser Assembly unsichtbar
// für COM-Komponenten. Wenn Sie auf einen Typ in dieser Assembly von
// COM zugreifen müssen, legen Sie das ComVisible-Attribut für diesen Typ auf "true" fest.
[assembly: ComVisible(false)]

// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird
[assembly: Guid("3fb8cecd-71b6-4a2c-8e7c-886e49be8936")]

// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten:
//
// Hauptversion
// Nebenversion
// Buildnummer
// Revision
//
// Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern
// übernehmen, indem Sie "*" eingeben:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
45 changes: 45 additions & 0 deletions Tokenizer/TokenType.cs
@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Tokenizer
{
/// <summary>
/// wrapper to hide implementation type of token
/// </summary>
public class TokenType
{
public Int32 Type { get; set; }


// override object.Equals
public override bool Equals(object obj)
{
//
// See the full list of guidelines at
// http://go.microsoft.com/fwlink/?LinkID=85237
// and also the guidance for operator== at
// http://go.microsoft.com/fwlink/?LinkId=85238
//

if (obj == null || GetType() != obj.GetType())
{
return false;
}

return Type == ((TokenType)obj).Type;

}

public override int GetHashCode()
{
return Type.GetHashCode();
}

public override string ToString()
{
return "TokenType:" + Type;
}
}
}
15 changes: 8 additions & 7 deletions CTokenizer/TokenWrapper.cs → Tokenizer/TokenWrapper.cs
@@ -1,21 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using Antlr.Runtime;

namespace CTokenizer
namespace Tokenizer
{
/// <summary>
/// wrapper around IToken to override Equals.
/// wrapper around ANTLR's IToken to override Equals.
/// Needed to make it work with GSTToken / GSTAlgorithm
/// </summary>
public class TokenWrapper
{
/// <summary>
/// the saved token object from ANTLR lexers
/// </summary>
public IToken Token { get; private set; }
private IToken Token { get; set; }

private string Text { get; set; }

private TokenType Type { get; set; }

public TokenWrapper(IToken token)
{
Expand Down
58 changes: 58 additions & 0 deletions Tokenizer/Tokenizer.csproj
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{044DABC4-13F0-4581-A9AF-FF645FD460EB}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Tokenizer</RootNamespace>
<AssemblyName>Tokenizer</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Antlr3.Runtime">
<HintPath>..\ThirdPartyLibs\Antlr3.Runtime.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="TokenType.cs" />
<Compile Include="TokenWrapper.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

0 comments on commit 65c622f

Please sign in to comment.