diff --git a/README.md b/README.md index f225dd0..1fe8ed1 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,70 @@ TRANSLATION_WEBSOCKET=ws://150.254.78.132:443/translate Uruchomienie serwera backend ``` -chmod u+x install.sh -./install.sh +cd backend +python3 manage.py runserver ``` Uruchomienie serwera marian-server na środowisku karty graficznej na porcie 443 ``` sudo ./tools/marian/build/marian-server --port 443 -m model/model.npz -v model/vocab.ende.yml model/vocab.ende.yml -``` \ No newline at end of file +``` + +Dodanie stage w Jenkinsfile: + +``` +pipeline { + + agent { + docker { + image 'loxygen/autozoil' + } + } + + + stages { + + stage ('build-pdf') { + steps { + withEnv(["HOME=${env.WORKSPACE}"]) { + sh """ + +make clean +make +""" + }} + + post { + always { + archiveArtifacts artifacts: 'msc-wojciech-jarmosz.pdf', fingerprint: true + archiveArtifacts artifacts: 'stats.txt', fingerprint: true + } + } + + } + + stage ('autozoil') { + steps { + withEnv(["HOME=${env.WORKSPACE}"]) { + sh """ +rm -f autozoil*.xml +make autozoil +""" + }} + + post { + always { + archiveArtifacts artifacts: 'autozoil*.xml', fingerprint: true + junit testResults: 'autozoil-external-check.xml', skipPublishingChecks: true, allowEmptyResults: true +// junit testResults: 'autozoil.xml' + } + } + } + + + + } +} +``` + +Skopiowanie plików z folderu helpers do repozytorium z pracą, która ma być poprawiana: diff --git a/helpers/autozoil-via-docker.sh b/helpers/autozoil-via-docker.sh new file mode 100644 index 0000000..437ee4e --- /dev/null +++ b/helpers/autozoil-via-docker.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -v $(pwd):/link -it loxygen/autozoil /opt/autozoil/autozoil --locale pl_PL /link/main.tex --alt-log-file /link/msc-zofia-fras.log diff --git a/helpers/contribution-declaration.tex.tmpl b/helpers/contribution-declaration.tex.tmpl new file mode 100644 index 0000000..a03d521 --- /dev/null +++ b/helpers/contribution-declaration.tex.tmpl @@ -0,0 +1,88 @@ + +\documentclass[a4paper]{article} +\usepackage{booktabs} +\usepackage{bibentry} +%\usepackage[style=verbose]{biblatex} +\usepackage[total={170mm,277mm}, left=20mm, top=10mm]{geometry} +\thispagestyle{empty} +\renewcommand{\labelitemi}{--} +\usepackage{tabularx} +\usepackage{longtable} + +\usepackage[backend=biber,maxnames=12]{biblatex} +\bibliography{../bibliography.bib} + + +\begin{document} + +\begin{flushright} + {{tday}} +\end{flushright} + +\medskip + +\begin{center} + {\bf Declaration} +\end{center} + +\bigskip + +I hereby declare that the contribution to the following paper: + +\smallskip + +\noindent\fullcite{this-paper} + +\smallskip + +\noindent is correctly characterized in the table below{% if nb_of_levels == 0 %}.{% elif nb_of_levels == 1 %} ({{ level_symbols[0] }} denotes equal contributions).{% else %} ({% for lix in range(nb_of_levels) %}{% if lix == nb_of_levels - 1 %}{} and {{ level_symbols[lix] }}{} denote groups of equal contributions).{% elif lix == 0 %}{{ level_symbols[lix] }}{% else %}, {{ level_symbols[lix] }}{% endif %}{% endfor %}{% endif %} +\medskip + + + +\begin{center} +\begin{longtable}{lc} +\toprule +\addlinespace[0.05cm] +Contributor & Description of main tasks \\ +\addlinespace[0.05cm] +\midrule +\addlinespace[0.1cm] +{% for author in data['authors'] %} +{{ author['author'] }}{% if 'level' in author %}{{level_symbols[author['level']-1]}}{% endif %} +& +\begin{minipage} [t] {0.6\textwidth} +\begin{itemize} +\itemsep=0em +{% for citem in author['contributions'] %} +\item {{citem}} +{% endfor %} +\end{itemize} +\end{minipage} \\ +{% if loop.index == loop.length %} +\addlinespace[0.25cm] +{% else %} +\addlinespace[0.5cm] +{% endif %} +{% endfor %} +\bottomrule +\end{longtable} +\end{center} + +\vskip 0.8cm + +\begin{tabularx}{\textwidth}{XXXX} +{% for author in data['authors'] %} +{{ author['author'] }} +{% if loop.index == loop.length %} +\\ +{% elif loop.index % 4 == 0 %} +\\ \addlinespace[0.5cm] +{% else %} +& +{% endif %} +{% endfor %} +\end{tabularx} + +\end{document} + diff --git a/helpers/flatten-structure.pl b/helpers/flatten-structure.pl new file mode 100644 index 0000000..356d2a5 --- /dev/null +++ b/helpers/flatten-structure.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl + +# Flatten file structure (needed for an arxiv package) + +use utf8; +use strict; + +binmode(STDIN, ':utf8'); +binmode(STDOUT, ':utf8'); + +my $subdir_regexp = qr{(?:figures|images)}; + +sub fix_subdirs { + my ($p) = @_; + + $p =~ s{/}{-}g; + + return $p; +} + +while (my $line=<>) { + $line =~ s<\\graphicspath\{(\s*)\{${subdir_regexp}/\}(\s*)><\\graphicspath\{$1\{.\}$2>; + $line =~ s<(\\includegraphics[^{}]*?\{)./${subdir_regexp}/><${1}./>g; + $line =~ s<(\\includegraphics[^{}]*?\{)([^\.][^{}]+/)><"$1".fix_subdirs($2)>ge; + $line =~ s<\\input\{inputs/><\\input\{inputs->g; + $line =~ s<\\minput\{scores/\#1\.txt\}><\\minput{scores-#1.txt}>g; + print $line; +} diff --git a/helpers/generate-contribution-declaration.py b/helpers/generate-contribution-declaration.py new file mode 100644 index 0000000..33a9457 --- /dev/null +++ b/helpers/generate-contribution-declaration.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import yaml +import sys +import os +import datetime + + +from jinja2 import FileSystemLoader, Environment, PackageLoader, select_autoescape +env = Environment( + loader=FileSystemLoader(os.path.abspath('.')), + autoescape=select_autoescape() +) + + +contributions = yaml.safe_load(sys.stdin) + + +tday = datetime.date.today() +ftday = tday.strftime('%B %d, %Y') + +level_symbols = [ + '*', + '\\dag', + '\\ddag', + '\\S', + '\\P', + '\\#'] + +nb_of_levels = len(set((c['level'] for c in contributions['authors'] if 'level' in c))) + + +template = env.get_template("contribution-declaration.tex.tmpl") + +print(template.render(tday=ftday, data=contributions, level_symbols=level_symbols, nb_of_levels=nb_of_levels)) diff --git a/helpers/generate-pdf-from-arxiv-package.sh b/helpers/generate-pdf-from-arxiv-package.sh new file mode 100644 index 0000000..2117a08 --- /dev/null +++ b/helpers/generate-pdf-from-arxiv-package.sh @@ -0,0 +1,32 @@ +#!/bin/bash -xe + +arxiv_package="$1" +target_pdf="$2" + +gentmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'gentmpdir') + +cp "$arxiv_package" "$gentmpdir/" + +cd "$gentmpdir" + +tar xvf "$arxiv_package" + +for f in * +do + if [[ -d "$f" ]] + then + echo >&2 "Unexpected directory: '$f'" + exit 1 + fi +done + +main_source=ms +main_source_file="${main_source}.tex" + +pdflatex "$main_source_file" +pdflatex "$main_source_file" +pdflatex "$main_source_file" + +cd - + +cp "$gentmpdir/ms.pdf" "$target_pdf" diff --git a/helpers/get-sentences.sh b/helpers/get-sentences.sh new file mode 100644 index 0000000..a462187 --- /dev/null +++ b/helpers/get-sentences.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +input_file="$1" +method="$2" + +extract_text() { + if [[ "$method" == "from-tex" ]] + then + detex "$input_file" | egrep '\S' | grep -v 'unsrt' | perl -pne 's/^\s+| +$//g' + else + bash helpers/pdf-to-plain-text.sh "$input_file" | perl helpers/strip-references.pl | perl -pne 'chomp $_; $_.=" "' + fi +} + +extract_text | python3 -m syntok.segmenter | egrep '\S' diff --git a/helpers/install-hooks.sh b/helpers/install-hooks.sh new file mode 100644 index 0000000..aee894e --- /dev/null +++ b/helpers/install-hooks.sh @@ -0,0 +1,4 @@ +#!/bin/bash -xe + +cp hooks/pre-push .git/hooks/pre-push +chmod 700 .git/hooks/pre-push diff --git a/helpers/pdf-to-plain-text.sh b/helpers/pdf-to-plain-text.sh new file mode 100644 index 0000000..c47adff --- /dev/null +++ b/helpers/pdf-to-plain-text.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pdftotext "$1" - | fgrep -v 'Confidential Review Copy' | grep -P -v '^(ACL 2020 Submission \*\*\*\. Confidential Review Copy\. DO NOT DISTRIBUTE\.|Anonymous ACL submission|Abstract|Results|Conclusions|https?://\S+)\s*$' | grep '[^[:space:]]' | egrep '[a-zA-Z]{2}' | perl -pne 's/\f//g;' | uniq diff --git a/helpers/prepare-arxiv-package.sh b/helpers/prepare-arxiv-package.sh new file mode 100644 index 0000000..9acbe39 --- /dev/null +++ b/helpers/prepare-arxiv-package.sh @@ -0,0 +1,149 @@ +#!/bin/bash -xe + +if command -v arxiv_latex_cleaner > /dev/null 2>/dev/null; +then + : +else + echo >&2 "Please install arxiv_latex_cleaner" + echo >&2 " git clone https://github.com/google-research/arxiv-latex-cleaner" + echo >&2 " cd arxiv_latex_cleaner" + echo >&2 " python3 setup.py install" + echo >&2 " (you might need to install zlib1g-dev and libjpeg8-dev packages first)" + exit 1 +fi + +latex_template="$1" +package_file="$2" + +packtmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'packtmpdir') +project_dir="$packtmpdir/project" +project_arxiv_dir="$packtmpdir/project_arXiv" +mkdir -p "$packtmpdir/project" + +arxiv_main_file=ms.tex + +copy_to_project() +{ + for f in "$@" + do + if [[ -r "$project_dir/$f" ]] + then + echo >&2 "The file '$f' will be overwritten. Something is wrong!!!" + exit 1 + fi + done + cp "$@" "$project_dir/" +} + +handle_subdir() +{ + subdir="$1" + + if [[ -d "$subdir" ]] + then + find $subdir -type f | while read f + do + nf=$(echo "$f" | perl -pne 's{^[^/]+/}{}; s{/}{-}') + dst="$project_dir/$subdir-$nf" + if [[ "$f" == *.tex ]] + then + perl helpers/flatten-structure.pl < "$f" > "$dst" + else + cp $f $dst + fi + done + fi +} + +handle_subdir images +handle_subdir figures +handle_subdir scores +handle_subdir inputs + +for latex_file in *.tex +do + if [[ "$latex_file" == "$arxiv_main_file" ]] + then + echo >&2 "Your repo cannot contain '$arxiv_main_file' file." + echo >&2 "This file is restricted for arxiv purposes" + exit 1 + fi + + target_file="$project_dir/$latex_file" + + if [[ "$latex_file" == "msc-zofia-fras.tex" ]] + then + target_file="$project_dir/$arxiv_main_file" + fi + + perl helpers/flatten-structure.pl < "$latex_file" > "$target_file" +done + +if [[ "$latex_template" == "vanilla" ]] +then + : +elif [[ "$latex_template" == "tacl" ]] +then + copy_to_project tacl2018v2.sty acl_natbib.bst +elif [[ "$latex_template" == "acl" ]] +then + copy_to_project acl.sty acl_natbib.bst +elif [[ "$latex_template" == "nle" ]] +then + copy_to_project nle.cls nlelike.bst cup_logo.eps cup_logo-eps-converted-to.pdf +elif [[ "$latex_template" == "sigconf" ]] +then + copy_to_project acmart.cls ACM-Reference-Format.bst +elif [[ "$latex_template" == "pw-thesis" ]] +then + : +elif [[ "$latex_template" == "coling" ]] +then + copy_to_project coling2020.sty coling.bst +elif [[ "$latex_template" == "llncs" ]] +then + copy_to_project llncs.cls splncs04.bst +elif [[ "$latex_template" == "emnlp" ]] +then + copy_to_project emnlp2020.sty acl_natbib.bst +elif [[ "$latex_template" == "eacl" ]] +then + copy_to_project eacl2021.sty acl_natbib.bst +elif [[ "$latex_template" == "neurips" ]] +then + copy_to_project neurips_2021.sty +elif [[ "$latex_template" == "icml" ]] +then + copy_to_project algorithm.sty \ + algorithmic.sty \ + fancyhdr.sty \ + icml2022.bst \ + icml2022.sty +elif [[ "$latex_template" == "ieee-access" ]] +then + copy_to_project bullet.png ieeeaccess.cls IEEEtran.cls ieeeaccess.cls spotcolor.sty logo.png notaglinelogo.png +elif [[ "$latex_template" == "ieee-conf" ]] +then + : +elif [[ "$latex_template" == "poleval" ]] +then + copy_to_project poleval.bst poleval.cls +else + echo >&2 "Unknown template '$latex_template'!!!" + exit 1 +fi + +copy_to_project bibliography.bib + +perl -pne 's/^\s*\\usepackage\{xurl\}$//' -i "$project_dir/extras.tex" + +heredir=`pwd` + +main_source=ms +main_source_file="${main_source}.tex" + +(cd "$project_dir" && pdflatex "$main_source_file" && bibtex "$main_source" && rm "${main_source}.pdf") + +arxiv_latex_cleaner "$project_dir" --commands_to_delete '\todo' +perl -pne 's/\\bibliography\{bibliography\}/\\bibliography\{ms\}/' -i "$project_arxiv_dir/$main_source_file" +(cd "$project_arxiv_dir" && tar zvcf "$heredir/$package_file" .) diff --git a/helpers/run-gitlab-runner.sh b/helpers/run-gitlab-runner.sh new file mode 100644 index 0000000..e2300c8 --- /dev/null +++ b/helpers/run-gitlab-runner.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +if [[ "$TEMPLATE_REPO_URL" == "" ]] +then + echo >&2 "Set the TEMPLATE_REPO_URL variable!" + exit 1 +fi + +echo 'gitlab-runner launched locally does not handle `include` directives.' +echo 'THIS IS AN UGLY WORK-AROUND FOR https://gitlab.com/gitlab-org/gitlab-runner/issues/3327' + +template_version=$(grep -P "^\s*ref:\s*'[0-9.]+'\s*$" .gitlab-ci.yml | grep -o -P '[0-9.]+') + +cookiecutter_paper_dir=`mktemp -d -t run-gitlab-runner.XXXXXXXXXXX` +(cd $cookiecutter_paper_dir && git clone ${TEMPLATE_REPO_DIR} -b $template_version) + +new_gitlab_ci_yml=`mktemp -t new-gitlab-ci-yml.XXXXXXXXX` + +perl -ne 'print if 1../THIS IS AN UGLY WORK-AROUND/' < .gitlab-ci.yml > $new_gitlab_ci_yml +cat ${cookiecutter_paper_dir}/cookiecutter-*-paper/main.yml >> $new_gitlab_ci_yml + +cp $new_gitlab_ci_yml .gitlab-ci.yml + +current_commit=`git rev-parse HEAD` + +# committing the manufactured .gitlab-ci.yml for a moment +git add .gitlab-ci.yml +git commit -m 'TEMPORARY COMMIT. DO NOT MERGE' + +gitlab-runner exec docker "$@" + +# going back to the original commit +git reset --hard $current_commit diff --git a/helpers/set-up-overleaf.sh b/helpers/set-up-overleaf.sh new file mode 100644 index 0000000..bd41692 --- /dev/null +++ b/helpers/set-up-overleaf.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e + +echo "Setting up interoperability with Overleaf..." +echo "Note that you need ability to clone repos from Overleaf." + +original_git_url=$(git remote get-url origin) + +echo "Make sure $original_git_url exists at your git server." +echo "It should be created as en empty repo, *uncheck* option" +echo "'Initialize repository with a README' or similar." + +echo "" +echo "Press ENTER to start" +read + +here_dir=$(basename $PWD) + + +make +make source-pack + +source_pack=msc-zofia-fras.zip + +echo "" +echo "Go to Overleaf, click New Project / Upload Project," +echo "then upload $source_pack from here ($(pwd)/$source_pack)" + +echo "" +echo "When it is uploaded, click Menu / Git, copy the URL" +echo "(something like https://git.overleaf.com/123456789ec0830001af836f)," +echo "paste here and press ENTER" + +read +overleaf_git_url=$REPLY +# remove "git clone" in case it was directly copied from Overleaf +overleaf_git_url=${overleaf_git_url#git clone } +overleaf_git_id=$(basename $overleaf_git_url) + + +cd .. + +backup_dir=${here_dir}-backup +mv $here_dir ${here_dir}-backup + +git clone $overleaf_git_url $here_dir + +cd $here_dir + +git remote add overleaf $overleaf_git_url +git remote set-url origin $original_git_url + +git config credential.helper "cache --timeout=10000000" + +cp ../$backup_dir/.cookiecutter.yml . + +git add .cookiecutter.yml +git commit -m 'Back cookiecutter config' + +cookiecutter https://git.wmi.amu.edu.pl/filipg/paper-cutter.git --checkout 5.3.1 --output-dir .. --config-file .cookiecutter.yml --no-input --overwrite-if-exists + +git add --all +git commit -m 'Bring back all the files' + +perl -pne 's{OVERLEAF_GIT_URL=}{OVERLEAF_GIT_URL='$overleaf_git_url'}' -i helpers/vars + +git add helpers/vars +git commit -m 'Set vars' + +git push overleaf master +git push origin master + +cd .. + +echo "DONE" +echo "Left ${backup_dir}, now type 'cd ..' and you can remove it" diff --git a/helpers/stats.sh b/helpers/stats.sh new file mode 100644 index 0000000..9783c52 --- /dev/null +++ b/helpers/stats.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +here_dir=$(dirname "$0") + +pdf_file="$1" + +if [[ "$pdf_file" == "" ]] +then + >&2 echo "no file given" + exit 1 +fi + +perl -e 'print "PHYSICAL PAGES\tCHARACTERS WITH SPACES\tSTANDARD PAGES\n"' +pages=$(pdfinfo "$1" | perl -ne 'print "$1\n" if /Pages:\s+(\d+)/') +chars=$(bash $here_dir/pdf-to-plain-text.sh "$1" | wc -m) +spages=$(echo "scale=1; $chars / 1800.0" | bc) +echo "$pages $chars $spages" diff --git a/helpers/strip-references.pl b/helpers/strip-references.pl new file mode 100644 index 0000000..10a3e28 --- /dev/null +++ b/helpers/strip-references.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl + +use strict; + +my @previously_skipped = (); + +my $was_reference = 0; + +while (my $line=) { + chomp $line; + + if ($line =~ /^References$/) { + if ($was_reference) { + for my $pline (@previously_skipped) { + print "$pline\n"; + } + @previously_skipped = (); + } else { + $was_reference = 1; + } + } + + if ($was_reference) { + push @previously_skipped, $line; + } else { + print "$line\n"; + } +} diff --git a/helpers/synchro.sh b/helpers/synchro.sh new file mode 100644 index 0000000..b245bc8 --- /dev/null +++ b/helpers/synchro.sh @@ -0,0 +1,24 @@ +#!/bin/bash -xe + +. helpers/vars + +if [[ "$OVERLEAF_GIT_URL" == "" ]] +then + >&2 echo "Please set OVERLEAF_GIT_URL in `helpers/vars`" + exit 1 +fi + +if [[ -e .git && ! -z "$(git status --untracked-files=no --porcelain)" ]]; then + >&2 echo "Uncommited changes, commit your changes first" + exit 1 +fi + +git remote add overleaf "$OVERLEAF_GIT_URL" || git remote set-url overleaf "$OVERLEAF_GIT_URL" + +git config credential.helper "cache --timeout=10000000" + +git pull origin master +git pull overleaf master + +git push overleaf master +git push origin master diff --git a/helpers/vars b/helpers/vars new file mode 100644 index 0000000..90bbd32 --- /dev/null +++ b/helpers/vars @@ -0,0 +1,2 @@ +# to be included with . +OVERLEAF_GIT_URL=https://git.overleaf.com/622792b464c39c99624ee794