Add necessary helper files and Jenkins stage definition

README.md

@@ -22,11 +22,70 @@ TRANSLATION_WEBSOCKET=ws://150.254.78.132:443/translate
 
 Uruchomienie serwera backend 
 ```
-chmod u+x install.sh
-./install.sh
+cd backend
+python3 manage.py runserver 
 ```
 
 Uruchomienie serwera marian-server na środowisku karty graficznej na porcie 443
 ```
 sudo ./tools/marian/build/marian-server --port 443 -m model/model.npz -v model/vocab.ende.yml model/vocab.ende.yml
-```
+```
+
+Dodanie stage w Jenkinsfile:
+
+```
+pipeline {
+
+    agent {
+        docker {
+            image 'loxygen/autozoil'
+        }
+    }
+
+
+    stages {
+
+	stage ('build-pdf') {
+            steps {
+              withEnv(["HOME=${env.WORKSPACE}"]) {
+                          sh """
+
+make clean
+make
+"""
+            }}
+
+            post {
+                always {
+                    archiveArtifacts artifacts: 'msc-wojciech-jarmosz.pdf', fingerprint: true
+                    archiveArtifacts artifacts: 'stats.txt', fingerprint: true
+                }
+            }
+
+	}
+
+	stage ('autozoil') {
+            steps {
+              withEnv(["HOME=${env.WORKSPACE}"]) {
+              sh """
+rm -f autozoil*.xml
+make autozoil
+"""
+            }}
+
+            post {
+                always {
+                    archiveArtifacts artifacts: 'autozoil*.xml', fingerprint: true
+                    junit testResults: 'autozoil-external-check.xml',  skipPublishingChecks: true, allowEmptyResults: true
+//                    junit testResults: 'autozoil.xml'
+                }
+            }
+	}
+
+
+
+    }
+}
+```
+
+Skopiowanie plików z folderu helpers do repozytorium z pracą, która ma być poprawiana:

helpers/autozoil-via-docker.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run -v $(pwd):/link  -it loxygen/autozoil /opt/autozoil/autozoil --locale pl_PL /link/main.tex --alt-log-file /link/msc-zofia-fras.log

helpers/contribution-declaration.tex.tmpl

@@ -0,0 +1,88 @@
+
+\documentclass[a4paper]{article}
+\usepackage{booktabs}
+\usepackage{bibentry}
+%\usepackage[style=verbose]{biblatex}
+\usepackage[total={170mm,277mm}, left=20mm, top=10mm]{geometry}
+\thispagestyle{empty}
+\renewcommand{\labelitemi}{--}
+\usepackage{tabularx}
+\usepackage{longtable}
+
+\usepackage[backend=biber,maxnames=12]{biblatex}
+\bibliography{../bibliography.bib}
+
+
+\begin{document}
+
+\begin{flushright}
+  {{tday}}
+\end{flushright}
+
+\medskip
+
+\begin{center}
+  {\bf Declaration}
+\end{center}
+
+\bigskip
+
+I hereby declare that the contribution to the following paper:
+
+\smallskip
+
+\noindent\fullcite{this-paper}
+
+\smallskip
+
+\noindent is correctly characterized in the table below{% if nb_of_levels == 0 %}.{% elif nb_of_levels == 1 %} ({{ level_symbols[0] }} denotes equal contributions).{% else %} ({% for lix in range(nb_of_levels) %}{% if lix == nb_of_levels - 1 %}{} and {{ level_symbols[lix] }}{} denote groups of equal contributions).{% elif lix == 0 %}{{ level_symbols[lix] }}{% else %}, {{ level_symbols[lix] }}{% endif %}{% endfor %}{% endif %}
+\medskip
+
+
+
+\begin{center}
+\begin{longtable}{lc}
+\toprule
+\addlinespace[0.05cm]
+Contributor & Description of main tasks \\
+\addlinespace[0.05cm]
+\midrule
+\addlinespace[0.1cm]
+{% for author in data['authors'] %}
+{{ author['author'] }}{% if 'level' in author %}{{level_symbols[author['level']-1]}}{% endif %}
+&
+\begin{minipage} [t] {0.6\textwidth}
+\begin{itemize}
+\itemsep=0em
+{% for citem in author['contributions'] %}
+\item {{citem}}
+{% endfor %}
+\end{itemize}
+\end{minipage} \\
+{% if loop.index == loop.length %}
+\addlinespace[0.25cm]
+{% else %}
+\addlinespace[0.5cm]
+{% endif %}
+{% endfor %}
+\bottomrule
+\end{longtable}
+\end{center}
+
+\vskip 0.8cm
+
+\begin{tabularx}{\textwidth}{XXXX}
+{% for author in data['authors'] %}
+{{ author['author'] }}
+{% if loop.index == loop.length %}
+\\
+{% elif loop.index % 4 == 0 %}
+\\ \addlinespace[0.5cm]
+{% else %}
+&
+{% endif %}
+{% endfor %}
+\end{tabularx}
+
+\end{document}
+

helpers/flatten-structure.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/perl
+
+# Flatten file structure (needed for an arxiv package)
+
+use utf8;
+use strict;
+
+binmode(STDIN, ':utf8');
+binmode(STDOUT, ':utf8');
+
+my $subdir_regexp = qr{(?:figures|images)};
+
+sub fix_subdirs {
+    my ($p) = @_;
+
+    $p =~ s{/}{-}g;
+
+    return $p;
+}
+
+while (my $line=<>) {
+    $line =~ s<\\graphicspath\{(\s*)\{${subdir_regexp}/\}(\s*)><\\graphicspath\{$1\{.\}$2>;
+    $line =~ s<(\\includegraphics[^{}]*?\{)./${subdir_regexp}/><${1}./>g;
+    $line =~ s<(\\includegraphics[^{}]*?\{)([^\.][^{}]+/)><"$1".fix_subdirs($2)>ge;
+    $line =~ s<\\input\{inputs/><\\input\{inputs->g;
+    $line =~ s<\\minput\{scores/\#1\.txt\}><\\minput{scores-#1.txt}>g;
+    print $line;
+}

helpers/generate-contribution-declaration.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import yaml
+import sys
+import os
+import datetime
+
+
+from jinja2 import FileSystemLoader, Environment, PackageLoader, select_autoescape
+env = Environment(
+    loader=FileSystemLoader(os.path.abspath('.')),
+    autoescape=select_autoescape()
+)
+
+
+contributions = yaml.safe_load(sys.stdin)
+
+
+tday = datetime.date.today()
+ftday = tday.strftime('%B %d, %Y')
+
+level_symbols = [
+    '*',
+    '\\dag',
+    '\\ddag',
+    '\\S',
+    '\\P',
+    '\\#']
+
+nb_of_levels = len(set((c['level'] for c in contributions['authors'] if 'level' in c)))
+
+
+template = env.get_template("contribution-declaration.tex.tmpl")
+
+print(template.render(tday=ftday, data=contributions, level_symbols=level_symbols, nb_of_levels=nb_of_levels))

helpers/generate-pdf-from-arxiv-package.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -xe
+
+arxiv_package="$1"
+target_pdf="$2"
+
+gentmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'gentmpdir')
+
+cp "$arxiv_package" "$gentmpdir/"
+
+cd "$gentmpdir"
+
+tar xvf "$arxiv_package"
+
+for f in *
+do
+    if [[ -d "$f" ]]
+    then
+        echo >&2 "Unexpected directory: '$f'"
+        exit 1
+    fi
+done
+
+main_source=ms
+main_source_file="${main_source}.tex"
+
+pdflatex "$main_source_file"
+pdflatex "$main_source_file"
+pdflatex "$main_source_file"
+
+cd -
+
+cp "$gentmpdir/ms.pdf" "$target_pdf"

helpers/get-sentences.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+input_file="$1"
+method="$2"
+
+extract_text() {
+    if [[ "$method" == "from-tex" ]]
+    then
+        detex "$input_file" | egrep '\S' | grep -v 'unsrt' | perl -pne 's/^\s+| +$//g'
+    else
+        bash helpers/pdf-to-plain-text.sh "$input_file" | perl helpers/strip-references.pl | perl -pne 'chomp $_; $_.=" "'
+    fi
+}
+
+extract_text | python3 -m syntok.segmenter | egrep '\S'

helpers/install-hooks.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -xe
+
+cp hooks/pre-push .git/hooks/pre-push
+chmod 700 .git/hooks/pre-push

helpers/pdf-to-plain-text.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+pdftotext "$1" - | fgrep -v 'Confidential Review Copy' | grep -P -v '^(ACL 2020 Submission \*\*\*\. Confidential Review Copy\. DO NOT DISTRIBUTE\.|Anonymous ACL submission|Abstract|Results|Conclusions|https?://\S+)\s*$' | grep '[^[:space:]]' | egrep '[a-zA-Z]{2}' | perl -pne 's/\f//g;' | uniq

helpers/prepare-arxiv-package.sh

@@ -0,0 +1,149 @@
+#!/bin/bash -xe
+
+if command -v arxiv_latex_cleaner > /dev/null 2>/dev/null;
+then
+    :
+else
+    echo >&2 "Please install arxiv_latex_cleaner"
+    echo >&2 "  git clone https://github.com/google-research/arxiv-latex-cleaner"
+    echo >&2 "  cd arxiv_latex_cleaner"
+    echo >&2 "  python3 setup.py install"
+    echo >&2 " (you might need to install zlib1g-dev and libjpeg8-dev packages first)"
+    exit 1
+fi
+
+latex_template="$1"
+package_file="$2"
+
+packtmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'packtmpdir')
+project_dir="$packtmpdir/project"
+project_arxiv_dir="$packtmpdir/project_arXiv"
+mkdir -p "$packtmpdir/project"
+
+arxiv_main_file=ms.tex
+
+copy_to_project()
+{
+    for f in "$@"
+    do
+        if [[ -r "$project_dir/$f" ]]
+        then
+            echo >&2 "The file '$f' will be overwritten. Something is wrong!!!"
+            exit 1
+        fi
+    done
+    cp "$@" "$project_dir/"
+}
+
+handle_subdir()
+{
+    subdir="$1"
+
+    if [[ -d "$subdir" ]]
+    then
+        find $subdir -type f | while read f
+        do
+            nf=$(echo "$f" | perl -pne 's{^[^/]+/}{}; s{/}{-}')
+            dst="$project_dir/$subdir-$nf"
+            if [[ "$f" == *.tex ]]
+            then
+                perl helpers/flatten-structure.pl < "$f" > "$dst"
+            else
+                cp $f $dst
+            fi
+        done
+    fi
+}
+
+handle_subdir images
+handle_subdir figures
+handle_subdir scores
+handle_subdir inputs
+
+for latex_file in *.tex
+do
+    if [[ "$latex_file" == "$arxiv_main_file" ]]
+    then
+        echo >&2 "Your repo cannot contain '$arxiv_main_file' file."
+        echo >&2 "This file is restricted for arxiv purposes"
+        exit 1
+    fi
+
+    target_file="$project_dir/$latex_file"
+
+    if [[ "$latex_file" == "msc-zofia-fras.tex" ]]
+    then
+        target_file="$project_dir/$arxiv_main_file"
+    fi
+
+    perl helpers/flatten-structure.pl < "$latex_file" > "$target_file"
+done
+
+if [[ "$latex_template" == "vanilla" ]]
+then
+    :
+elif [[ "$latex_template" == "tacl" ]]
+then
+    copy_to_project tacl2018v2.sty acl_natbib.bst
+elif [[ "$latex_template" == "acl" ]]
+then
+     copy_to_project acl.sty acl_natbib.bst
+elif [[ "$latex_template" == "nle" ]]
+then
+     copy_to_project nle.cls nlelike.bst cup_logo.eps cup_logo-eps-converted-to.pdf
+elif [[ "$latex_template" == "sigconf" ]]
+then
+     copy_to_project acmart.cls ACM-Reference-Format.bst
+elif [[ "$latex_template" == "pw-thesis" ]]
+then
+     :
+elif [[ "$latex_template" == "coling" ]]
+then
+     copy_to_project coling2020.sty coling.bst
+elif [[ "$latex_template" == "llncs" ]]
+then
+    copy_to_project llncs.cls splncs04.bst
+elif [[ "$latex_template" == "emnlp" ]]
+then
+     copy_to_project emnlp2020.sty acl_natbib.bst
+elif [[ "$latex_template" == "eacl" ]]
+then
+     copy_to_project eacl2021.sty acl_natbib.bst
+elif [[ "$latex_template" == "neurips" ]]
+then
+     copy_to_project neurips_2021.sty
+elif [[ "$latex_template" == "icml" ]]
+then
+     copy_to_project algorithm.sty \
+                     algorithmic.sty \
+                     fancyhdr.sty \
+                     icml2022.bst \
+                     icml2022.sty
+elif [[ "$latex_template" == "ieee-access" ]]
+then
+     copy_to_project bullet.png ieeeaccess.cls IEEEtran.cls ieeeaccess.cls spotcolor.sty logo.png notaglinelogo.png
+elif [[ "$latex_template" == "ieee-conf" ]]
+then
+    :
+elif [[ "$latex_template" == "poleval" ]]
+then
+     copy_to_project poleval.bst poleval.cls
+else
+    echo >&2 "Unknown template '$latex_template'!!!"
+    exit 1
+fi
+
+copy_to_project bibliography.bib
+
+perl -pne 's/^\s*\\usepackage\{xurl\}$//' -i "$project_dir/extras.tex"
+
+heredir=`pwd`
+
+main_source=ms
+main_source_file="${main_source}.tex"
+
+(cd "$project_dir" && pdflatex "$main_source_file" && bibtex "$main_source" && rm "${main_source}.pdf")
+
+arxiv_latex_cleaner "$project_dir" --commands_to_delete '\todo'
+perl -pne 's/\\bibliography\{bibliography\}/\\bibliography\{ms\}/' -i "$project_arxiv_dir/$main_source_file"
+(cd "$project_arxiv_dir" && tar zvcf "$heredir/$package_file" .)

helpers/run-gitlab-runner.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [[ "$TEMPLATE_REPO_URL" == "" ]]
+then
+    echo >&2 "Set the TEMPLATE_REPO_URL variable!"
+    exit 1
+fi
+
+echo 'gitlab-runner launched locally does not handle `include` directives.'
+echo 'THIS IS AN UGLY WORK-AROUND FOR https://gitlab.com/gitlab-org/gitlab-runner/issues/3327'
+
+template_version=$(grep -P "^\s*ref:\s*'[0-9.]+'\s*$" .gitlab-ci.yml | grep -o -P '[0-9.]+')
+
+cookiecutter_paper_dir=`mktemp -d -t run-gitlab-runner.XXXXXXXXXXX`
+(cd $cookiecutter_paper_dir && git clone ${TEMPLATE_REPO_DIR} -b $template_version)
+
+new_gitlab_ci_yml=`mktemp -t new-gitlab-ci-yml.XXXXXXXXX`
+
+perl -ne 'print if 1../THIS IS AN UGLY WORK-AROUND/' < .gitlab-ci.yml > $new_gitlab_ci_yml
+cat ${cookiecutter_paper_dir}/cookiecutter-*-paper/main.yml >> $new_gitlab_ci_yml
+
+cp $new_gitlab_ci_yml .gitlab-ci.yml
+
+current_commit=`git rev-parse HEAD`
+
+# committing the manufactured .gitlab-ci.yml for a moment
+git add .gitlab-ci.yml
+git commit -m 'TEMPORARY COMMIT. DO NOT MERGE'
+
+gitlab-runner exec docker "$@"
+
+# going back to the original commit
+git reset --hard $current_commit

helpers/set-up-overleaf.sh

@@ -0,0 +1,75 @@
+#!/bin/bash -e
+
+echo "Setting up interoperability with Overleaf..."
+echo "Note that you need ability to clone repos from Overleaf."
+
+original_git_url=$(git remote get-url origin)
+
+echo "Make sure $original_git_url exists at your git server."
+echo "It should be created as en empty repo, *uncheck* option"
+echo "'Initialize repository with a README' or similar."
+
+echo ""
+echo "Press ENTER to start"
+read
+
+here_dir=$(basename $PWD)
+
+
+make
+make source-pack
+
+source_pack=msc-zofia-fras.zip
+
+echo ""
+echo "Go to Overleaf, click New Project / Upload Project,"
+echo "then upload $source_pack from here ($(pwd)/$source_pack)"
+
+echo ""
+echo "When it is uploaded, click Menu / Git, copy the URL"
+echo "(something like https://git.overleaf.com/123456789ec0830001af836f),"
+echo "paste here and press ENTER"
+
+read
+overleaf_git_url=$REPLY
+# remove "git clone" in case it was directly copied from Overleaf
+overleaf_git_url=${overleaf_git_url#git clone }
+overleaf_git_id=$(basename $overleaf_git_url)
+
+
+cd ..
+
+backup_dir=${here_dir}-backup
+mv $here_dir ${here_dir}-backup
+
+git clone $overleaf_git_url $here_dir
+
+cd $here_dir
+
+git remote add overleaf $overleaf_git_url
+git remote set-url origin $original_git_url
+
+git config credential.helper "cache --timeout=10000000"
+
+cp ../$backup_dir/.cookiecutter.yml .
+
+git add .cookiecutter.yml
+git commit -m 'Back cookiecutter config'
+
+cookiecutter https://git.wmi.amu.edu.pl/filipg/paper-cutter.git --checkout 5.3.1 --output-dir .. --config-file .cookiecutter.yml --no-input --overwrite-if-exists
+
+git add --all
+git commit -m 'Bring back all the files'
+
+perl -pne 's{OVERLEAF_GIT_URL=}{OVERLEAF_GIT_URL='$overleaf_git_url'}' -i helpers/vars
+
+git add helpers/vars
+git commit -m 'Set vars'
+
+git push overleaf master
+git push origin master
+
+cd ..
+
+echo "DONE"
+echo "Left ${backup_dir}, now type 'cd ..' and you can remove it"

helpers/stats.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+here_dir=$(dirname "$0")
+
+pdf_file="$1"
+
+if [[ "$pdf_file" == "" ]]
+then
+    >&2 echo "no file given"
+    exit 1
+fi
+
+perl -e 'print "PHYSICAL PAGES\tCHARACTERS WITH SPACES\tSTANDARD PAGES\n"'
+pages=$(pdfinfo "$1" | perl -ne 'print "$1\n" if /Pages:\s+(\d+)/')
+chars=$(bash $here_dir/pdf-to-plain-text.sh "$1" | wc -m)
+spages=$(echo "scale=1; $chars / 1800.0" | bc)
+echo "$pages	$chars	$spages"

helpers/strip-references.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/perl
+
+use strict;
+
+my @previously_skipped = ();
+
+my $was_reference = 0;
+
+while (my $line=<STDIN>) {
+    chomp $line;
+
+    if ($line =~ /^References$/) {
+        if ($was_reference) {
+            for my $pline (@previously_skipped) {
+                print "$pline\n";
+            }
+            @previously_skipped = ();
+        } else {
+            $was_reference = 1;
+        }
+    }
+
+    if ($was_reference) {
+        push @previously_skipped, $line;
+    } else {
+        print "$line\n";
+    }
+}

helpers/synchro.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -xe
+
+. helpers/vars
+
+if [[ "$OVERLEAF_GIT_URL" == "" ]]
+then
+    >&2 echo "Please set OVERLEAF_GIT_URL in `helpers/vars`"
+    exit 1
+fi
+
+if [[ -e .git && ! -z "$(git status --untracked-files=no --porcelain)" ]]; then
+    >&2 echo "Uncommited changes, commit your changes first"
+    exit 1
+fi
+
+git remote add overleaf "$OVERLEAF_GIT_URL" || git remote set-url overleaf "$OVERLEAF_GIT_URL"
+
+git config credential.helper "cache --timeout=10000000"
+
+git pull origin master
+git pull overleaf master
+
+git push overleaf master
+git push origin master

helpers/vars

@@ -0,0 +1,2 @@
+# to be included with .
+OVERLEAF_GIT_URL=https://git.overleaf.com/622792b464c39c99624ee794