lemmagen sentence lemmatizer
This commit is contained in:
parent
e558cb05d8
commit
015a916d20
Binary file not shown.
BIN
mgiza-aligner/LemmaGenSentenceLemmatizer/LemmaGen/LemmaSharp.dll
Normal file
BIN
mgiza-aligner/LemmaGenSentenceLemmatizer/LemmaGen/LemmaSharp.dll
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
mgiza-aligner/LemmaGenSentenceLemmatizer/LemmaGen/Lzma#.dll
Normal file
BIN
mgiza-aligner/LemmaGenSentenceLemmatizer/LemmaGen/Lzma#.dll
Normal file
Binary file not shown.
@ -0,0 +1,22 @@
|
|||||||
|
|
||||||
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
|
# Visual Studio 14
|
||||||
|
VisualStudioVersion = 14.0.25420.1
|
||||||
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LemmaGenSentenceLemmatizer", "LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer.csproj", "{66B48B61-7304-4352-8720-DD664AD06138}"
|
||||||
|
EndProject
|
||||||
|
Global
|
||||||
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
Release|Any CPU = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
|
{66B48B61-7304-4352-8720-DD664AD06138}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{66B48B61-7304-4352-8720-DD664AD06138}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{66B48B61-7304-4352-8720-DD664AD06138}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{66B48B61-7304-4352-8720-DD664AD06138}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
|
HideSolutionNode = FALSE
|
||||||
|
EndGlobalSection
|
||||||
|
EndGlobal
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8" ?>
|
||||||
|
<configuration>
|
||||||
|
<startup>
|
||||||
|
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
|
||||||
|
</startup>
|
||||||
|
</configuration>
|
||||||
@ -0,0 +1,73 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
|
<PropertyGroup>
|
||||||
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
<ProjectGuid>{66B48B61-7304-4352-8720-DD664AD06138}</ProjectGuid>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||||
|
<RootNamespace>LemmaGenSentenceLemmatizer</RootNamespace>
|
||||||
|
<AssemblyName>LemmaGenSentenceLemmatizer</AssemblyName>
|
||||||
|
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
|
||||||
|
<FileAlignment>512</FileAlignment>
|
||||||
|
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||||
|
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||||
|
<DebugSymbols>true</DebugSymbols>
|
||||||
|
<DebugType>full</DebugType>
|
||||||
|
<Optimize>false</Optimize>
|
||||||
|
<OutputPath>bin\Debug\</OutputPath>
|
||||||
|
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||||
|
<ErrorReport>prompt</ErrorReport>
|
||||||
|
<WarningLevel>4</WarningLevel>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||||
|
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||||
|
<DebugType>pdbonly</DebugType>
|
||||||
|
<Optimize>true</Optimize>
|
||||||
|
<OutputPath>bin\Release\</OutputPath>
|
||||||
|
<DefineConstants>TRACE</DefineConstants>
|
||||||
|
<ErrorReport>prompt</ErrorReport>
|
||||||
|
<WarningLevel>4</WarningLevel>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Reference Include="LemmaSharp">
|
||||||
|
<HintPath>..\LemmaGen\LemmaSharp.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="LemmaSharpPrebuilt">
|
||||||
|
<HintPath>..\LemmaGen\LemmaSharpPrebuilt.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="LemmaSharpPrebuiltCompact">
|
||||||
|
<HintPath>..\LemmaGen\LemmaSharpPrebuiltCompact.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Lzma#">
|
||||||
|
<HintPath>..\LemmaGen\Lzma#.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System" />
|
||||||
|
<Reference Include="System.Core" />
|
||||||
|
<Reference Include="System.Xml.Linq" />
|
||||||
|
<Reference Include="System.Data.DataSetExtensions" />
|
||||||
|
<Reference Include="Microsoft.CSharp" />
|
||||||
|
<Reference Include="System.Data" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Xml" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Compile Include="Program.cs" />
|
||||||
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
|
<Compile Include="SentenceLemmatizer.cs" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="App.config" />
|
||||||
|
</ItemGroup>
|
||||||
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
|
<Target Name="BeforeBuild">
|
||||||
|
</Target>
|
||||||
|
<Target Name="AfterBuild">
|
||||||
|
</Target>
|
||||||
|
-->
|
||||||
|
</Project>
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|AnyCPU'">
|
||||||
|
<StartArguments>pl</StartArguments>
|
||||||
|
</PropertyGroup>
|
||||||
|
</Project>
|
||||||
@ -0,0 +1,32 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace LemmaGenSentenceLemmatizer
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static void Main(string[] args)
|
||||||
|
{
|
||||||
|
if (args.Length == 1)
|
||||||
|
{
|
||||||
|
SentenceLemmatizer lemmatizer = new SentenceLemmatizer(args[0]);
|
||||||
|
string line = Console.ReadLine();
|
||||||
|
while (!string.IsNullOrEmpty(line))
|
||||||
|
{
|
||||||
|
Console.WriteLine(lemmatizer.lemmatizeSentence(line));
|
||||||
|
line = Console.ReadLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Console.WriteLine("Wrong number of parameters. Please specify the language code as the only parameter");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
using System.Reflection;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
|
// General Information about an assembly is controlled through the following
|
||||||
|
// set of attributes. Change these attribute values to modify the information
|
||||||
|
// associated with an assembly.
|
||||||
|
[assembly: AssemblyTitle("LemmaGenSentenceLemmatizer")]
|
||||||
|
[assembly: AssemblyDescription("")]
|
||||||
|
[assembly: AssemblyConfiguration("")]
|
||||||
|
[assembly: AssemblyCompany("")]
|
||||||
|
[assembly: AssemblyProduct("LemmaGenSentenceLemmatizer")]
|
||||||
|
[assembly: AssemblyCopyright("Copyright © 2017")]
|
||||||
|
[assembly: AssemblyTrademark("")]
|
||||||
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
// Setting ComVisible to false makes the types in this assembly not visible
|
||||||
|
// to COM components. If you need to access a type in this assembly from
|
||||||
|
// COM, set the ComVisible attribute to true on that type.
|
||||||
|
[assembly: ComVisible(false)]
|
||||||
|
|
||||||
|
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||||
|
[assembly: Guid("66b48b61-7304-4352-8720-dd664ad06138")]
|
||||||
|
|
||||||
|
// Version information for an assembly consists of the following four values:
|
||||||
|
//
|
||||||
|
// Major Version
|
||||||
|
// Minor Version
|
||||||
|
// Build Number
|
||||||
|
// Revision
|
||||||
|
//
|
||||||
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
|
// by using the '*' as shown below:
|
||||||
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
|
[assembly: AssemblyVersion("1.0.0.0")]
|
||||||
|
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||||
@ -0,0 +1,81 @@
|
|||||||
|
using LemmaSharp;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace LemmaGenSentenceLemmatizer
|
||||||
|
{
|
||||||
|
class SentenceLemmatizer
|
||||||
|
{
|
||||||
|
private char[] wordInnerSeparator = { '-' };
|
||||||
|
|
||||||
|
private ILemmatizer lemmatizer;
|
||||||
|
|
||||||
|
public SentenceLemmatizer(string languageCode)
|
||||||
|
{
|
||||||
|
switch (languageCode)
|
||||||
|
{
|
||||||
|
case "pl":
|
||||||
|
lemmatizer = new LemmatizerPrebuiltCompact(LanguagePrebuilt.Polish);
|
||||||
|
break;
|
||||||
|
case "en":
|
||||||
|
lemmatizer = new LemmatizerPrebuiltCompact(LanguagePrebuilt.English);
|
||||||
|
break;
|
||||||
|
case "hr":
|
||||||
|
lemmatizer = new LemmatizerPrebuiltCompact(LanguagePrebuilt.Serbian);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new ArgumentException("Unknown language code: " + languageCode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public string lemmatizeSentence(string sentence)
|
||||||
|
{
|
||||||
|
string[] tokens = sentence.Split(null);
|
||||||
|
|
||||||
|
string result = "";
|
||||||
|
foreach (string token in tokens)
|
||||||
|
{
|
||||||
|
result += lemmatizeWord(token) + " ";
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private string lemmatizeWord(string word)
|
||||||
|
{
|
||||||
|
string result = "";
|
||||||
|
string[] parts = word.Split(wordInnerSeparator);
|
||||||
|
if (parts.Length == 2)
|
||||||
|
{
|
||||||
|
string firstPart = parts[0];
|
||||||
|
if (!parts[0].EndsWith("o"))
|
||||||
|
{
|
||||||
|
firstPart = lemmatizer.Lemmatize(firstPart);
|
||||||
|
}
|
||||||
|
string secondPart = lemmatizer.Lemmatize(parts[1]);
|
||||||
|
result = firstPart + "-" + secondPart;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result = lemmatizer.Lemmatize(word);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if (result == "" || result.Contains(" "))
|
||||||
|
{
|
||||||
|
return word;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8" ?>
|
||||||
|
<configuration>
|
||||||
|
<startup>
|
||||||
|
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
|
||||||
|
</startup>
|
||||||
|
</configuration>
|
||||||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8" ?>
|
||||||
|
<configuration>
|
||||||
|
<startup>
|
||||||
|
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
|
||||||
|
</startup>
|
||||||
|
</configuration>
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
|
||||||
|
<assemblyIdentity version="1.0.0.0" name="MyApplication.app"/>
|
||||||
|
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
|
||||||
|
<security>
|
||||||
|
<requestedPrivileges xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||||
|
<requestedExecutionLevel level="asInvoker" uiAccess="false"/>
|
||||||
|
</requestedPrivileges>
|
||||||
|
</security>
|
||||||
|
</trustInfo>
|
||||||
|
</assembly>
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,10 @@
|
|||||||
|
j:\documents\visual studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaGenSentenceLemmatizer.exe.config
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaGenSentenceLemmatizer.exe
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaGenSentenceLemmatizer.pdb
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaSharp.dll
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaSharpPrebuilt.dll
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\LemmaSharpPrebuiltCompact.dll
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\bin\Debug\Lzma#.dll
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\obj\Debug\LemmaGenSentenceLemmatizer.csprojResolveAssemblyReference.cache
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\obj\Debug\LemmaGenSentenceLemmatizer.exe
|
||||||
|
j:\Documents\Visual Studio 2015\Projects\LemmaGenSentenceLemmatizer\LemmaGenSentenceLemmatizer\obj\Debug\LemmaGenSentenceLemmatizer.pdb
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue
Block a user