From c6a71ddd00df9f35e935e55dd868b4e09aff6893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Jaworski?= Date: Wed, 22 Aug 2018 13:28:39 +0200 Subject: [PATCH 1/4] debug --- long-db-search.log | 389 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 long-db-search.log diff --git a/long-db-search.log b/long-db-search.log new file mode 100644 index 0000000..4066b5e --- /dev/null +++ b/long-db-search.log @@ -0,0 +1,389 @@ +2017-10-25 09:57:01.172 : Concordia server process start +2017-10-25 10:13:23.027 : Concordia server initiated successfully, waiting for requests +2017-10-25 10:18:44.067 : concordia search +2017-10-25 10:18:44.491 : got raw concordia result +2017-10-25 10:18:44.491 : _getResultFromFragment +2017-10-25 10:18:44.547 : got examples +2017-10-25 10:18:44.598 : got target fragments +2017-10-25 10:18:44.624 : got examples +2017-10-25 10:18:44.644 : got target fragments +2017-10-25 10:18:44.668 : got examples +2017-10-25 10:18:44.689 : got target fragments +2017-10-25 10:18:44.729 : got examples +2017-10-25 10:18:44.767 : got target fragments +2017-10-25 10:18:44.800 : got examples +2017-10-25 10:18:44.834 : got target fragments +2017-10-25 10:18:44.836 : got examples +2017-10-25 10:18:44.838 : got target fragments +2017-10-25 10:18:44.839 : got examples +2017-10-25 10:18:44.840 : got target fragments +2017-10-25 10:18:44.854 : got examples +2017-10-25 10:18:44.865 : got target fragments +2017-10-25 10:18:44.893 : got examples +2017-10-25 10:18:44.931 : got target fragments +2017-10-25 10:18:44.964 : got examples +2017-10-25 10:18:45.002 : got target fragments +2017-10-25 10:18:45.025 : got examples +2017-10-25 10:18:45.063 : got target fragments +2017-10-25 10:18:45.109 : got examples +2017-10-25 10:18:45.121 : got target fragments +2017-10-25 10:18:45.149 : got examples +2017-10-25 10:18:45.167 : got target fragments +2017-10-25 10:18:45.197 : got examples +2017-10-25 10:18:45.211 : got target fragments +2017-10-25 10:18:45.232 : got examples +2017-10-25 10:18:45.243 : got target fragments +2017-10-25 10:18:45.244 : _getResultFromFragment +2017-10-25 10:18:45.264 : got examples +2017-10-25 10:18:45.303 : got target fragments +2017-10-25 10:18:45.303 : got examples +2017-10-25 10:18:45.304 : got target fragments +2017-10-25 10:18:45.316 : got examples +2017-10-25 10:18:45.333 : got target fragments +2017-10-25 10:18:45.350 : got examples +2017-10-25 10:18:45.366 : got target fragments +2017-10-25 10:18:45.367 : _getResultFromFragment +2017-10-25 10:18:45.404 : got examples +2017-10-25 10:18:45.417 : got target fragments +2017-10-25 10:18:46.125 : got examples +2017-10-25 10:18:46.623 : got target fragments +2017-10-25 10:18:46.649 : got examples +2017-10-25 10:18:46.663 : got target fragments +2017-10-25 10:18:46.877 : got examples +2017-10-25 10:18:48.117 : got target fragments +2017-10-25 10:18:48.630 : got examples +2017-10-25 10:18:49.632 : got target fragments +2017-10-25 10:18:50.367 : got examples +2017-10-25 10:18:50.890 : got target fragments +2017-10-25 10:18:50.890 : got examples +2017-10-25 10:18:50.891 : got target fragments +2017-10-25 10:18:51.880 : got examples +2017-10-25 10:18:52.632 : got target fragments +2017-10-25 10:18:53.867 : got examples +2017-10-25 10:18:56.367 : got target fragments +2017-10-25 10:18:56.637 : got examples +2017-10-25 10:18:56.659 : got target fragments +2017-10-25 10:18:56.677 : got examples +2017-10-25 10:18:56.730 : got target fragments +2017-10-25 10:18:56.753 : got examples +2017-10-25 10:18:56.762 : got target fragments +2017-10-25 10:18:56.794 : got examples +2017-10-25 10:18:56.831 : got target fragments +2017-10-25 10:18:56.847 : got examples +2017-10-25 10:18:56.868 : got target fragments +2017-10-25 10:18:56.906 : got examples +2017-10-25 10:18:56.925 : got target fragments +2017-10-25 10:18:56.926 : _getResultFromFragment +2017-10-25 10:18:56.975 : got examples +2017-10-25 10:18:57.003 : got target fragments +2017-10-25 10:18:57.004 : _getResultFromFragment +2017-10-25 10:18:57.023 : got examples +2017-10-25 10:18:57.067 : got target fragments +2017-10-25 10:18:57.120 : got examples +2017-10-25 10:18:57.139 : got target fragments +2017-10-25 10:18:57.184 : got examples +2017-10-25 10:18:57.221 : got target fragments +2017-10-25 10:18:57.240 : got examples +2017-10-25 10:18:57.277 : got target fragments +2017-10-25 10:18:57.334 : got examples +2017-10-25 10:18:57.363 : got target fragments +2017-10-25 10:18:57.386 : got examples +2017-10-25 10:18:57.400 : got target fragments +2017-10-25 10:18:57.420 : got examples +2017-10-25 10:18:57.445 : got target fragments +2017-10-25 10:18:57.457 : got examples +2017-10-25 10:18:57.471 : got target fragments +2017-10-25 10:18:57.484 : got examples +2017-10-25 10:18:57.505 : got target fragments +2017-10-25 10:18:57.539 : got examples +2017-10-25 10:18:57.597 : got target fragments +2017-10-25 10:18:57.605 : got examples +2017-10-25 10:18:57.626 : got target fragments +2017-10-25 10:18:57.640 : got examples +2017-10-25 10:18:57.679 : got target fragments +2017-10-25 10:18:57.694 : got examples +2017-10-25 10:18:57.709 : got target fragments +2017-10-25 10:18:57.773 : got examples +2017-10-25 10:18:57.789 : got target fragments +2017-10-25 10:18:57.805 : got examples +2017-10-25 10:18:57.826 : got target fragments +2017-10-25 10:18:57.826 : _getResultFromFragment +2017-10-25 10:18:57.850 : got examples +2017-10-25 10:18:57.867 : got target fragments +2017-10-25 10:18:57.884 : got examples +2017-10-25 10:18:57.919 : got target fragments +2017-10-25 10:18:58.375 : got examples +2017-10-25 10:18:58.886 : got target fragments +2017-10-25 10:18:59.630 : got examples +2017-10-25 10:19:00.131 : got target fragments +2017-10-25 10:19:00.876 : got examples +2017-10-25 10:19:02.368 : got target fragments +2017-10-25 10:19:03.618 : got examples +2017-10-25 10:19:04.123 : got target fragments +2017-10-25 10:19:04.624 : got examples +2017-10-25 10:19:04.640 : got target fragments +2017-10-25 10:19:04.658 : got examples +2017-10-25 10:19:04.677 : got target fragments +2017-10-25 10:19:04.688 : got examples +2017-10-25 10:19:04.694 : got target fragments +2017-10-25 10:19:04.714 : got examples +2017-10-25 10:19:04.749 : got target fragments +2017-10-25 10:19:04.769 : got examples +2017-10-25 10:19:04.780 : got target fragments +2017-10-25 10:19:04.802 : got examples +2017-10-25 10:19:04.821 : got target fragments +2017-10-25 10:19:04.850 : got examples +2017-10-25 10:19:04.872 : got target fragments +2017-10-25 10:19:04.886 : got examples +2017-10-25 10:19:04.897 : got target fragments +2017-10-25 10:19:04.921 : got examples +2017-10-25 10:19:04.928 : got target fragments +2017-10-25 10:19:04.929 : _getResultFromFragment +2017-10-25 10:19:04.952 : got examples +2017-10-25 10:19:04.990 : got target fragments +2017-10-25 10:19:05.017 : got examples +2017-10-25 10:19:05.048 : got target fragments +2017-10-25 10:19:05.072 : got examples +2017-10-25 10:19:05.113 : got target fragments +2017-10-25 10:19:05.131 : got examples +2017-10-25 10:19:05.162 : got target fragments +2017-10-25 10:19:05.631 : got examples +2017-10-25 10:19:06.373 : got target fragments +2017-10-25 10:19:07.618 : got examples +2017-10-25 10:19:08.135 : got target fragments +2017-10-25 10:19:08.878 : got examples +2017-10-25 10:19:09.378 : got target fragments +2017-10-25 10:19:10.382 : got examples +2017-10-25 10:19:10.410 : got target fragments +2017-10-25 10:19:10.420 : got examples +2017-10-25 10:19:10.439 : got target fragments +2017-10-25 10:19:10.463 : got examples +2017-10-25 10:19:10.474 : got target fragments +2017-10-25 10:19:10.488 : got examples +2017-10-25 10:19:10.505 : got target fragments +2017-10-25 10:19:10.545 : got examples +2017-10-25 10:19:10.568 : got target fragments +2017-10-25 10:19:10.593 : got examples +2017-10-25 10:19:10.612 : got target fragments +2017-10-25 10:19:10.631 : got examples +2017-10-25 10:19:10.638 : got target fragments +2017-10-25 10:19:10.657 : got examples +2017-10-25 10:19:10.663 : got target fragments +2017-10-25 10:19:10.663 : _getResultFromFragment +2017-10-25 10:19:10.685 : got examples +2017-10-25 10:19:10.707 : got target fragments +2017-10-25 10:19:10.721 : got examples +2017-10-25 10:19:10.743 : got target fragments +2017-10-25 10:19:10.743 : got examples +2017-10-25 10:19:10.750 : got target fragments +2017-10-25 10:19:10.779 : got examples +2017-10-25 10:19:10.791 : got target fragments +2017-10-25 10:19:10.822 : got examples +2017-10-25 10:19:10.846 : got target fragments +2017-10-25 10:19:10.870 : got examples +2017-10-25 10:19:10.897 : got target fragments +2017-10-25 10:19:10.917 : got examples +2017-10-25 10:19:10.932 : got target fragments +2017-10-25 10:19:10.956 : got examples +2017-10-25 10:19:10.967 : got target fragments +2017-10-25 10:19:10.997 : got examples +2017-10-25 10:19:11.021 : got target fragments +2017-10-25 10:19:11.041 : got examples +2017-10-25 10:19:11.050 : got target fragments +2017-10-25 10:19:11.075 : got examples +2017-10-25 10:19:11.095 : got target fragments +2017-10-25 10:19:11.122 : got examples +2017-10-25 10:19:11.164 : got target fragments +2017-10-25 10:19:11.183 : got examples +2017-10-25 10:19:11.191 : got target fragments +2017-10-25 10:19:11.214 : got examples +2017-10-25 10:19:11.239 : got target fragments +2017-10-25 10:19:11.261 : got examples +2017-10-25 10:19:11.281 : got target fragments +2017-10-25 10:19:11.282 : _getResultFromFragment +2017-10-25 10:19:11.311 : got examples +2017-10-25 10:19:11.335 : got target fragments +2017-10-25 10:19:11.357 : got examples +2017-10-25 10:19:11.371 : got target fragments +2017-10-25 10:19:11.625 : got examples +2017-10-25 10:19:13.118 : got target fragments +2017-10-25 10:19:14.868 : got examples +2017-10-25 10:19:14.884 : got target fragments +2017-10-25 10:19:14.906 : got examples +2017-10-25 10:19:14.915 : got target fragments +2017-10-25 10:19:14.932 : got examples +2017-10-25 10:19:14.942 : got target fragments +2017-10-25 10:19:14.968 : got examples +2017-10-25 10:19:14.990 : got target fragments +2017-10-25 10:19:15.041 : got examples +2017-10-25 10:19:15.054 : got target fragments +2017-10-25 10:19:15.086 : got examples +2017-10-25 10:19:15.104 : got target fragments +2017-10-25 10:19:15.129 : got examples +2017-10-25 10:19:15.156 : got target fragments +2017-10-25 10:19:15.173 : got examples +2017-10-25 10:19:15.184 : got target fragments +2017-10-25 10:19:15.207 : got examples +2017-10-25 10:19:15.223 : got target fragments +2017-10-25 10:19:15.238 : got examples +2017-10-25 10:19:15.287 : got target fragments +2017-10-25 10:19:15.306 : got examples +2017-10-25 10:19:15.323 : got target fragments +2017-10-25 10:19:15.344 : got examples +2017-10-25 10:19:15.360 : got target fragments +2017-10-25 10:19:15.360 : _getResultFromFragment +2017-10-25 10:19:15.387 : got examples +2017-10-25 10:19:15.414 : got target fragments +2017-10-25 10:19:15.442 : got examples +2017-10-25 10:19:15.468 : got target fragments +2017-10-25 10:19:15.495 : got examples +2017-10-25 10:19:15.517 : got target fragments +2017-10-25 10:19:15.545 : got examples +2017-10-25 10:19:15.567 : got target fragments +2017-10-25 10:19:15.586 : got examples +2017-10-25 10:19:15.624 : got target fragments +2017-10-25 10:19:15.650 : got examples +2017-10-25 10:19:15.671 : got target fragments +2017-10-25 10:19:15.672 : got examples +2017-10-25 10:19:15.679 : got target fragments +2017-10-25 10:19:15.695 : got examples +2017-10-25 10:19:15.703 : got target fragments +2017-10-25 10:19:15.734 : got examples +2017-10-25 10:19:15.755 : got target fragments +2017-10-25 10:19:15.768 : got examples +2017-10-25 10:19:15.789 : got target fragments +2017-10-25 10:19:15.808 : got examples +2017-10-25 10:19:15.829 : got target fragments +2017-10-25 10:19:15.845 : got examples +2017-10-25 10:19:15.877 : got target fragments +2017-10-25 10:19:15.925 : got examples +2017-10-25 10:19:15.954 : got target fragments +2017-10-25 10:19:16.003 : got examples +2017-10-25 10:19:16.032 : got target fragments +2017-10-25 10:19:16.053 : got examples +2017-10-25 10:19:16.075 : got target fragments +2017-10-25 10:19:16.075 : _getResultFromFragment +2017-10-25 10:19:16.102 : got examples +2017-10-25 10:19:16.187 : got target fragments +2017-10-25 10:19:16.220 : got examples +2017-10-25 10:19:16.270 : got target fragments +2017-10-25 10:19:16.294 : got examples +2017-10-25 10:19:16.339 : got target fragments +2017-10-25 10:19:16.358 : got examples +2017-10-25 10:19:16.389 : got target fragments +2017-10-25 10:19:16.414 : got examples +2017-10-25 10:19:16.444 : got target fragments +2017-10-25 10:19:16.468 : got examples +2017-10-25 10:19:16.497 : got target fragments +2017-10-25 10:19:16.515 : got examples +2017-10-25 10:19:16.532 : got target fragments +2017-10-25 10:19:16.552 : got examples +2017-10-25 10:19:16.563 : got target fragments +2017-10-25 10:19:16.588 : got examples +2017-10-25 10:19:16.610 : got target fragments +2017-10-25 10:19:16.632 : got examples +2017-10-25 10:19:16.660 : got target fragments +2017-10-25 10:19:16.682 : got examples +2017-10-25 10:19:16.717 : got target fragments +2017-10-25 10:19:16.750 : got examples +2017-10-25 10:19:16.769 : got target fragments +2017-10-25 10:19:16.789 : got examples +2017-10-25 10:19:16.818 : got target fragments +2017-10-25 10:19:16.837 : got examples +2017-10-25 10:19:16.864 : got target fragments +2017-10-25 10:19:16.881 : got examples +2017-10-25 10:19:16.908 : got target fragments +2017-10-25 10:19:16.908 : _getResultFromFragment +2017-10-25 10:19:16.920 : got examples +2017-10-25 10:19:16.938 : got target fragments +2017-10-25 10:19:16.960 : got examples +2017-10-25 10:19:16.975 : got target fragments +2017-10-25 10:19:16.995 : got examples +2017-10-25 10:19:17.017 : got target fragments +2017-10-25 10:19:17.033 : got examples +2017-10-25 10:19:17.073 : got target fragments +2017-10-25 10:19:17.100 : got examples +2017-10-25 10:19:17.124 : got target fragments +2017-10-25 10:19:17.142 : got examples +2017-10-25 10:19:17.151 : got target fragments +2017-10-25 10:19:17.188 : got examples +2017-10-25 10:19:17.195 : got target fragments +2017-10-25 10:19:17.215 : got examples +2017-10-25 10:19:17.225 : got target fragments +2017-10-25 10:19:17.253 : got examples +2017-10-25 10:19:17.273 : got target fragments +2017-10-25 10:19:17.300 : got examples +2017-10-25 10:19:17.319 : got target fragments +2017-10-25 10:19:17.340 : got examples +2017-10-25 10:19:17.363 : got target fragments +2017-10-25 10:19:17.393 : got examples +2017-10-25 10:19:17.420 : got target fragments +2017-10-25 10:19:17.455 : got examples +2017-10-25 10:19:17.468 : got target fragments +2017-10-25 10:19:17.510 : got examples +2017-10-25 10:19:17.538 : got target fragments +2017-10-25 10:19:17.554 : got examples +2017-10-25 10:19:17.601 : got target fragments +2017-10-25 10:19:17.601 : _getResultFromFragment +2017-10-25 10:19:17.633 : got examples +2017-10-25 10:19:17.646 : got target fragments +2017-10-25 10:19:17.678 : got examples +2017-10-25 10:19:17.704 : got target fragments +2017-10-25 10:19:17.728 : got examples +2017-10-25 10:19:17.743 : got target fragments +2017-10-25 10:19:17.759 : got examples +2017-10-25 10:19:17.768 : got target fragments +2017-10-25 10:19:17.794 : got examples +2017-10-25 10:19:17.807 : got target fragments +2017-10-25 10:19:17.808 : got examples +2017-10-25 10:19:17.810 : got target fragments +2017-10-25 10:19:17.822 : got examples +2017-10-25 10:19:17.836 : got target fragments +2017-10-25 10:19:17.850 : got examples +2017-10-25 10:19:17.877 : got target fragments +2017-10-25 10:19:17.906 : got examples +2017-10-25 10:19:18.869 : got target fragments +2017-10-25 10:19:19.935 : got examples +2017-10-25 10:19:21.143 : got target fragments +2017-10-25 10:19:21.882 : got examples +2017-10-25 10:19:22.876 : got target fragments +2017-10-25 10:19:23.626 : got examples +2017-10-25 10:19:24.871 : got target fragments +2017-10-25 10:19:26.123 : got examples +2017-10-25 10:19:26.871 : got target fragments +2017-10-25 10:19:27.127 : got examples +2017-10-25 10:19:27.649 : got target fragments +2017-10-25 10:19:27.650 : got examples +2017-10-25 10:19:27.650 : got target fragments +2017-10-25 10:19:27.651 : _getResultFromFragment +2017-10-25 10:19:27.670 : got examples +2017-10-25 10:19:27.685 : got target fragments +2017-10-25 10:19:27.686 : got examples +2017-10-25 10:19:27.693 : got target fragments +2017-10-25 10:19:27.709 : got examples +2017-10-25 10:19:27.726 : got target fragments +2017-10-25 10:19:27.752 : got examples +2017-10-25 10:19:27.771 : got target fragments +2017-10-25 10:19:27.789 : got examples +2017-10-25 10:19:27.806 : got target fragments +2017-10-25 10:19:27.808 : got examples +2017-10-25 10:19:27.809 : got target fragments +2017-10-25 10:19:27.824 : got examples +2017-10-25 10:19:27.839 : got target fragments +2017-10-25 10:19:27.855 : got examples +2017-10-25 10:19:27.866 : got target fragments +2017-10-25 10:19:28.630 : got examples +2017-10-25 10:19:29.618 : got target fragments +2017-10-25 10:19:30.626 : got examples +2017-10-25 10:19:31.628 : got target fragments +2017-10-25 10:19:36.869 : got examples +2017-10-25 10:19:37.408 : got target fragments +2017-10-25 10:19:37.427 : got examples +2017-10-25 10:19:37.454 : got target fragments +2017-10-25 10:19:37.455 : got examples +2017-10-25 10:19:37.457 : got target fragments +2017-10-25 10:19:37.484 : got examples +2017-10-25 10:19:37.489 : got target fragments +2017-10-25 10:19:37.489 : got examples +2017-10-25 10:19:37.499 : got target fragments +2017-10-25 10:19:37.500 : got db result From 8e1d2d2c42fcdd43d65bac4d535172eb26350ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Jaworski?= Date: Wed, 22 Aug 2018 13:30:03 +0200 Subject: [PATCH 2/4] debug --- .gitignore | 1 + cat/versions_available/stocznia_plen.cfg | 2 +- concordia-server/logger.cpp | 2 +- concordia-server/searcher_controller.cpp | 5 +++++ concordia-server/unit_dao.cpp | 3 +++ tests/addLemmatizedTM.sh | 6 +++--- 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index cdae01b..8824f97 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ mgiza-aligner/mgiza/mgizapp/src/mkcls/Makefile mgiza-aligner/mgiza/mgizapp/src/mkcls/cmake_install.cmake __pycache__ import-requests/request_handler.log +import-requests/request_handler.pid mgiza-aligner/corpus-compilator/corpora mgiza-aligner/corpus-compilator/dictionaries/*lem mgiza-aligner/corpus-compilator/dictionary-compilator/*.tsv diff --git a/cat/versions_available/stocznia_plen.cfg b/cat/versions_available/stocznia_plen.cfg index 83239a6..75ab960 100644 --- a/cat/versions_available/stocznia_plen.cfg +++ b/cat/versions_available/stocznia_plen.cfg @@ -1,5 +1,5 @@ dir@#@stocznia_plen -concordia_host@#@localhost +concordia_host@#@concordia.poleng concordia_port@#@8800 tmid@#@2 desc@#@Witamy w interaktywnym demo systemu Concordia. System znajduje najdłuższe fragmenty zdania wejściowego w pamięci tłumaczeń. Proszę wpisać polskie zdanie w poniższe pole i nacisnąć Enter (albo użyć przycisku "search"). Aby zapoznać się z systemem możesz użyć wcześniej przygotowanych przykładów - po prostu kliknij link "apply" przy wybranym przykładzie. Po wyszukaniu, kliknij na wybrany podświetlony fragment, aby zobaczyć jego kontekst. diff --git a/concordia-server/logger.cpp b/concordia-server/logger.cpp index 0fd0949..1b349b6 100644 --- a/concordia-server/logger.cpp +++ b/concordia-server/logger.cpp @@ -67,7 +67,7 @@ void Logger::logConcordiaSearchResult(const ConcordiaSearchResult & result) { void Logger::_initialize(log4cpp::Category & root) { log4cpp::Appender *appender = new log4cpp::FileAppender("default", LOG_FILE_PATH); log4cpp::PatternLayout *layout = new log4cpp::PatternLayout(); - layout->setConversionPattern("%d{%Y-%m-%d %H:%M:%S}%c %x: %m%n"); + layout->setConversionPattern("%d{%Y-%m-%d %H:%M:%S.%l}%c %x: %m%n"); appender->setLayout(layout); root.setPriority(log4cpp::Priority::INFO); diff --git a/concordia-server/searcher_controller.cpp b/concordia-server/searcher_controller.cpp index ca7b1c6..3b617f1 100644 --- a/concordia-server/searcher_controller.cpp +++ b/concordia-server/searcher_controller.cpp @@ -128,13 +128,18 @@ void SearcherController::concordiaPhraseSearch(rapidjson::Writer & jsonWriter, std::string & pattern, const int tmId) { + Logger::log("concordia search"); + boost::ptr_map::iterator it = _concordiasMap->find(tmId); if (it != _concordiasMap->end()) { TokenizedSentence originalPattern = it->second->tokenize(pattern, false, false); std::string lemmatizedPattern = _lemmatizerFacade->lemmatizeIfNeeded(originalPattern.getTokenizedSentence(), tmId); boost::shared_ptr rawConcordiaResult = it->second->concordiaSearch(lemmatizedPattern, true); + Logger::log("got raw concordia result"); + CompleteConcordiaSearchResult result = _unitDAO.getConcordiaResult(rawConcordiaResult, originalPattern); + Logger::log("got db result"); jsonWriter.StartObject(); jsonWriter.String("status"); diff --git a/concordia-server/unit_dao.cpp b/concordia-server/unit_dao.cpp index eca6f5b..85370b3 100644 --- a/concordia-server/unit_dao.cpp +++ b/concordia-server/unit_dao.cpp @@ -92,6 +92,7 @@ SimpleSearchResult UnitDAO::_getResultFromFragment( const MatchedPatternFragment & fragment, const TokenizedSentence & tokenizedPattern) { + Logger::log("_getResultFromFragment"); DBconnection connection; connection.startTransaction(); @@ -112,6 +113,7 @@ SimpleSearchResult UnitDAO::_getResultFromFragment( params.push_back(new IntParam(2*(sOccurence.getOffset()+fragment.getMatchedLength()))); params.push_back(new IntParam(sOccurence.getId())); PGresult * result = connection.execute(query, params); + Logger::log("got examples"); ExampleOccurence occurence(connection.getIntValue(result,0,0), // example id connection.getIntValue(result,0,3), // matched example start connection.getIntValue(result,0,4), // matched example end @@ -129,6 +131,7 @@ SimpleSearchResult UnitDAO::_getResultFromFragment( targetParams.push_back(new IntParam(sOccurence.getOffset())); targetParams.push_back(new IntParam(sOccurence.getOffset() + fragment.getMatchedLength() - 1)); PGresult * targetResult = connection.execute(targetQuery, targetParams); + Logger::log("got target fragments"); int prevPos = -2; int currStart = -1; diff --git a/tests/addLemmatizedTM.sh b/tests/addLemmatizedTM.sh index e411375..8a83148 100755 --- a/tests/addLemmatizedTM.sh +++ b/tests/addLemmatizedTM.sh @@ -1,7 +1,7 @@ #!/bin/sh -CORPUS_NAME="logofag_enpl" -SRC_LANG_ID=2 -TRG_LANG_ID=1 +CORPUS_NAME="stocznia_plen" +SRC_LANG_ID=1 +TRG_LANG_ID=2 ./addAlignedLemmatizedTM.py $CORPUS_NAME ../mgiza-aligner/corpora/$CORPUS_NAME/src_final.txt $SRC_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/trg_final.txt $TRG_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/aligned_final.txt From 7d3fa9e2ae31ae20420ed889b7edec2316cf1769 Mon Sep 17 00:00:00 2001 From: rjawor Date: Wed, 29 Aug 2018 12:56:47 +0200 Subject: [PATCH 3/4] installation on poleng computer, clean up --- .gitignore | 3 +++ INSTALL.txt | 2 +- cat/versions_available/europarl_sample.cfg | 14 +++++++------- cat/versions_enabled/europarl_sample.cfg | 1 + mgiza-aligner/Makefile | 6 +++--- tests/addLemmatizedTM.sh | 6 +++--- 6 files changed, 18 insertions(+), 14 deletions(-) create mode 120000 cat/versions_enabled/europarl_sample.cfg diff --git a/.gitignore b/.gitignore index 8824f97..f5b2913 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ logs/concordia-server.log logs/pgbouncer.log +pgbouncer.log logs/phrase-searches.json concordia.cfg concordia-server/config.hpp @@ -40,3 +41,5 @@ import-requests/request_handler.pid mgiza-aligner/corpus-compilator/corpora mgiza-aligner/corpus-compilator/dictionaries/*lem mgiza-aligner/corpus-compilator/dictionary-compilator/*.tsv +mgiza-aligner/mgiza/mgizapp/inst/ +mgiza-aligner/mgiza/mgizapp/install_manifest.txt diff --git a/INSTALL.txt b/INSTALL.txt index 49f6848..66f10fb 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -6,7 +6,7 @@ - grant all privileges on database concordia_server to concordia; - sudo vim /etc/postgresql/9.3/main/pg_hba.conf: change "local all all peer" to "local all all md5" - pgbouncer: - - sudo apt-get install autoconf automake m4 libtool pkg-config libevent-dev autogen + - sudo apt-get install autoconf automake m4 libtool pkg-config libevent-dev autogen python-docutils $ git clone https://github.com/pgbouncer/pgbouncer.git $ cd pgbouncer $ git submodule init diff --git a/cat/versions_available/europarl_sample.cfg b/cat/versions_available/europarl_sample.cfg index 9494336..d6a4004 100644 --- a/cat/versions_available/europarl_sample.cfg +++ b/cat/versions_available/europarl_sample.cfg @@ -1,10 +1,10 @@ dir@#@europarl_sample concordia_host@#@localhost concordia_port@#@8800 -tmid@#@1 -desc@#@Europarl sample (1000 sentences) -enjoy@#@Życzymy udanej pracy z systemem! -prompt@#@Wprowadź zdanie (po polsku): -suggestion@#@Na każde państwo członkowskie Unii Europejskiej przypada jeden komisarz. -suggestion@#@Komisja Europejska przygotowuje raport na najbliższym posiedzeniu. -suggestion@#@Wspólny Komitet przyjmuje swój statut. +tmid@#@2 +desc@#@Welcome to the interactive Concordia demo. The system finds the longest matches of the pattern sentence in its translation memory. This translation memory is over 1.5M sentences taken from English-Polish corpus of European Law (Europarl + JRC-Acquis). Please enter an English sentence in the field below and press Enter (or use the search button). This instance of Concordia works best with law sentences, but is very likely to output some results for any English sentence. You can also use predefined samples, simply use the link "show/hide samples" and apply one of the sample sentences. After the search, click on the highlighted fragments to see their context. +enjoy@#@Enjoy your work with the system! +prompt@#@Enter search pattern (English sentence): +suggestion@#@Every ship in the European Union must have a crew of 50 or more workers. +suggestion@#@It is impossible to abolish the customs duties on fruit and vegetables. +suggestion@#@The convention on human rights was held in Geneva. diff --git a/cat/versions_enabled/europarl_sample.cfg b/cat/versions_enabled/europarl_sample.cfg new file mode 120000 index 0000000..c90ed2e --- /dev/null +++ b/cat/versions_enabled/europarl_sample.cfg @@ -0,0 +1 @@ +../versions_available/europarl_sample.cfg \ No newline at end of file diff --git a/mgiza-aligner/Makefile b/mgiza-aligner/Makefile index 8ad6958..23df30a 100644 --- a/mgiza-aligner/Makefile +++ b/mgiza-aligner/Makefile @@ -1,9 +1,9 @@ -SRC_LANG=pl -TRG_LANG=en +SRC_LANG=en +TRG_LANG=pl CORPUS_NAME=europarl_sample SEPARATOR=@\#@ -DICTIONARY_WEIGHT=0 +DICTIONARY_WEIGHT=3 all: corpora/$(CORPUS_NAME)/aligned_final.txt corpora/$(CORPUS_NAME)/src_final.txt corpora/$(CORPUS_NAME)/trg_final.txt diff --git a/tests/addLemmatizedTM.sh b/tests/addLemmatizedTM.sh index 8a83148..8cb99b0 100755 --- a/tests/addLemmatizedTM.sh +++ b/tests/addLemmatizedTM.sh @@ -1,7 +1,7 @@ #!/bin/sh -CORPUS_NAME="stocznia_plen" -SRC_LANG_ID=1 -TRG_LANG_ID=2 +CORPUS_NAME="europarl_sample" +SRC_LANG_ID=2 +TRG_LANG_ID=1 ./addAlignedLemmatizedTM.py $CORPUS_NAME ../mgiza-aligner/corpora/$CORPUS_NAME/src_final.txt $SRC_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/trg_final.txt $TRG_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/aligned_final.txt From 44905f3a6167a612a5c3547990f9748cdac47c6d Mon Sep 17 00:00:00 2001 From: rjawor Date: Wed, 29 Aug 2018 13:50:08 +0200 Subject: [PATCH 4/4] removed debug --- concordia-server/searcher_controller.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/concordia-server/searcher_controller.cpp b/concordia-server/searcher_controller.cpp index 3b617f1..67268bf 100644 --- a/concordia-server/searcher_controller.cpp +++ b/concordia-server/searcher_controller.cpp @@ -128,18 +128,14 @@ void SearcherController::concordiaPhraseSearch(rapidjson::Writer & jsonWriter, std::string & pattern, const int tmId) { - Logger::log("concordia search"); - boost::ptr_map::iterator it = _concordiasMap->find(tmId); if (it != _concordiasMap->end()) { TokenizedSentence originalPattern = it->second->tokenize(pattern, false, false); std::string lemmatizedPattern = _lemmatizerFacade->lemmatizeIfNeeded(originalPattern.getTokenizedSentence(), tmId); boost::shared_ptr rawConcordiaResult = it->second->concordiaSearch(lemmatizedPattern, true); - Logger::log("got raw concordia result"); CompleteConcordiaSearchResult result = _unitDAO.getConcordiaResult(rawConcordiaResult, originalPattern); - Logger::log("got db result"); jsonWriter.StartObject(); jsonWriter.String("status");