rjawor/concordia-server
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new cat sample, remote tests

Browse Source
This commit is contained in:
rjawor 2015-09-01 09:34:27 +02:00
parent 22923fb426
commit 68d0fc5d7f
10 changed files with 330 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cat/cat.html
View File

@ -1,24 +0,0 @@
<html>
<head>
<script src="js/jquery-1.11.3.min.js"></script>
<script src="js/cat.js"></script>
<link rel="stylesheet" href="css/iatagger.css" />
<meta charset="UTF-8">
</head>
<body>
<div id="header">
</div>
<div id="content">
<img src="images/banner.jpg" alt="Banner" />
<br/><br/><br/>
<label for="searchInput">Enter search pattern (sentence):</label>
<input id="searchInput" type="text" value="Każde państwo unii europejskiej dba o dobro obywateli i służbę zdrowia." />
<br/><br/>
<input type="button" value="search" onclick="searchHandle()" />
<br/><br/><br/><br/>
<div id="result">
</div>
</div>
</body>
</html>

10
cat/css/iatagger.css
View File

@ -21,6 +21,16 @@
padding:0;
}
.suggestionsInvisible {
display:none;
}
.suggestion {
text-decoration: underline;
font-weight: bold;
color:#003d4c;
cursor:pointer;
}
.fragmentDetails {
border-style: solid;

33
cat/index.html Normal file
View File

@ -0,0 +1,33 @@
<html>
<head>
<script src="js/jquery-1.11.3.min.js"></script>
<script src="js/cat.js"></script>
<link rel="stylesheet" href="css/iatagger.css" />
<meta charset="UTF-8">
</head>
<body>
<div id="header">
</div>
<div id="content">
<a href="http://tmconcordia.sourceforge.net/" target="_blank"><img src="images/banner.jpg" alt="Banner" /></a>
<br/><br/><br/>
<label for="searchInput">Enter search pattern (sentence):</label>
<span class="suggestion" onclick="showHideSuggestions()">show/hide samples</span>
<br/><br/>
<div class="suggestionsInvisible" id="suggestions">
<ul>
<li>Every ship in the European Union must have a crew of 50 or more workers. <span class="suggestion" onclick="searchText('Every ship in the European Union must have a crew of 50 or more workers.');">apply</span></li>
<li>It is impossible to abolish the customs duties on fruit and vegetables. <span class="suggestion" onclick="searchText('It is impossible to abolish the customs duties on fruit and vegetables.');">apply</span></li>
</ul>
<br/><br/>
</div>
<input id="searchInput" type="text" value="" />
<br/><br/>
<input type="button" value="search" onclick="searchHandle()" />
<br/><br/><br/><br/>
<div id="result">
</div>
</div>
</body>
</html>

13
cat/js/cat.js
View File

@ -1,3 +1,5 @@
var concordiaUrl = 'http://localhost';
$(document).ready(function() {
$('#searchInput').bind("enterKey",function(e){
searchHandle();
@ -16,7 +18,7 @@ function searchHandle() {
}
$.ajax({
url: 'http://localhost',
url: concordiaUrl,
type: 'post',
dataType: 'json',
success: function (data) {
@ -80,3 +82,12 @@ function displayDetails(caller, number) {
$('.fragmentDetails').css('display', 'none');
$('#fragment'+number).css('display', 'block');
}
function searchText(text) {
$("#searchInput").val(text);
searchHandle();
}
function showHideSuggestions() {
$('#suggestions').toggleClass('suggestionsInvisible');
}

102
doc/install.txt Normal file
View File

@ -0,0 +1,102 @@
- sudo apt-get install postgresql libfcgi-dev libpq-dev
- clone github repo, mkdir build, cd build, ../cmake.sh, make
- sudo -u postgres psql postgres
- create user concordia with encrypted password 'concordia';
- CREATE DATABASE concordia_server ENCODING 'UTF8' OWNER concordia TEMPLATE template0;
- grant all privileges on database concordia_server to concordia;
- sudo vim /etc/postgresql/9.3/main/pg_hba.conf: change "local all all peer" to "local all all md5"
- pgbouncer: wget https://pgbouncer.github.io/downloads/files/1.6/pgbouncer-1.6.tar.gz
- sudo apt-get install libevent-dev
- ./db/recreateDb.sh
- nginx:
sudo -s
nginx=stable # use nginx=development for latest development version
add-apt-repository ppa:nginx/$nginx
apt-get update
apt-get install nginx
sites-available:
cat_html:
# Default server configuration
#
server {
listen 80 default_server;
listen [::]:80 default_server;
# SSL configuration
#
# listen 443 ssl default_server;
# listen [::]:443 ssl default_server;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
root /var/www/html;
rename default to fcgi_concordia
fcgi_concordia:
server {
listen 8800 default_server;
listen [::]:8800 default_server;
# SSL configuration
#
# listen 443 ssl default_server;
# listen [::]:443 ssl default_server;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
fastcgi_pass 127.0.0.1:8000;
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;
}
- add links in sites-enabled, sudo service nginx restart
- install cat html to /var/www/html (adjust ajax requests)
- sudo apt-get install spawn-fcgi
- mkdir index
- ./db/startPGbouncer.sh
- ./scripts/restart.sh

73
tests/remote/addFile.py Executable file
View File

@ -0,0 +1,73 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import json
import urllib2
import sys
import time
BUFFER_SIZE = 500
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def add_data(data):
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data)).read()
sourceFile = sys.argv[1]
targetFile = sys.argv[2]
tmId = int(sys.argv[3])
totalLines = file_len(sourceFile)
if file_len(targetFile) != totalLines:
print "File lengths do not match"
sys.exit(1)
data = {
'operation': 'addSentences'
}
sentences = []
start = time.time()
with open(sys.argv[1]) as sourceSentences:
with open(sys.argv[2]) as targetSentences:
lineNumber = 0
for sourceSentence in sourceSentences:
lineNumber += 1
targetSentence = targetSentences.readline()
sentences.append([tmId, sourceSentence, targetSentence])
if lineNumber % BUFFER_SIZE == 0:
data['sentences'] = sentences
sentences = []
add_data(data)
mark = time.time()
print "Added %d of %d sentences. Time elapsed: %.4f s, current speed: %.4f sentences/second" % (lineNumber, totalLines, mark-start, lineNumber/(mark-start))
if len(sentences) > 0:
data['sentences'] = sentences
add_data(data)
end = time.time()
print "Added all %d sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (lineNumber, end-start, lineNumber/(end-start))
print "Generating index..."
start = time.time()
data = {
'operation': 'refreshIndex'
}
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data)).read()
end = time.time()
print "Index regeneration complete. The operation took %.4f s" % (end - start)

27
tests/remote/addSentence.py Executable file
View File

@ -0,0 +1,27 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import json
import urllib2
import sys
import time
data = {
'operation': 'addSentence',
'sourceSentence':sys.argv[1],
'targetSentence':sys.argv[2],
'tmId':sys.argv[3]
}
start = time.time()
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
response = json.loads(urllib2.urlopen(req, json.dumps(data)).read())
end = time.time()
print "Execution time: %.4f seconds." % (end-start)
print "Result: "
print response

25
tests/remote/concordiaSearch.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import json
import urllib2
import sys
import time
data = {
'operation': 'concordiaSearch',
'pattern':sys.argv[1]
}
start = time.time()
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
response = json.loads(urllib2.urlopen(req, json.dumps(data)).read())
end = time.time()
print "Execution time: %.4f seconds." % (end-start)
print "Result: "
print response

23
tests/remote/generateIndex.py Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import json
import urllib2
import sys
import time
print "Generating index..."
start = time.time()
data = {
'operation': 'refreshIndex'
}
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data)).read()
end = time.time()
print "Index regeneration complete. The operation took %.4f s" % (end - start)

25
tests/remote/simpleSearch.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import unittest
import json
import urllib2
import sys
import time
data = {
'operation': 'simpleSearch',
'pattern':sys.argv[1]
}
start = time.time()
req = urllib2.Request('http://concordia.vm.wmi.amu.edu.pl:8800/')
req.add_header('Content-Type', 'application/json')
response = json.loads(urllib2.urlopen(req, json.dumps(data)).read())
end = time.time()
print "Execution time: %.4f seconds." % (end-start)
print "Result: "
print response