From 8ac8e869c12d3a40b4087a63f8d477d9e8f3c709 Mon Sep 17 00:00:00 2001 From: dominik24c Date: Thu, 19 May 2022 18:15:11 +0200 Subject: [PATCH] update - route of uploading students list - add additional handling with potential errors --- backend/app/__init__.py | 4 +++ backend/app/base/utils.py | 6 ++++ backend/app/config.py | 4 +++ backend/app/coordinator/routes/students.py | 36 ++++++++++++++++------ backend/app/coordinator/utils.py | 28 ++++++++--------- backend/app/errors.py | 7 +++++ 6 files changed, 60 insertions(+), 25 deletions(-) create mode 100644 backend/app/errors.py diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 4139fd7..c867d98 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -9,6 +9,7 @@ from .dependencies import db, ma from .commands.startapp import startapp from .utils import import_models from .api import api_bp +from .errors import request_entity_too_large def create_app(config_name: str = None) -> Flask: @@ -35,4 +36,7 @@ def create_app(config_name: str = None) -> Flask: # register commands app.cli.add_command(startapp) + # register errors + app.register_error_handler(413, request_entity_too_large) + return app diff --git a/backend/app/base/utils.py b/backend/app/base/utils.py index 867072e..a4487f2 100644 --- a/backend/app/base/utils.py +++ b/backend/app/base/utils.py @@ -1,5 +1,6 @@ from typing import TypedDict, Tuple +from flask import current_app from flask_sqlalchemy import BaseQuery from sqlalchemy import desc @@ -36,3 +37,8 @@ def paginate_models(page: str, query: BaseQuery) -> PaginationResponse or Tuple[ 'items': query.items, 'max_pages': query.pages } + + +def is_allowed_extensions(filename: str): + return '.' in filename and \ + filename.rsplit('.', 1)[1].lower() in current_app.config['ALLOWED_EXTENSIONS'] diff --git a/backend/app/config.py b/backend/app/config.py index 1ce0e7b..9176821 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -8,8 +8,12 @@ class Config: BASE_DIR = Path(__file__).resolve().parent.parent SRC_DIR = BASE_DIR / "app" EXCLUDED_DIRS = ["__pycache__", "commands"] + ENABLE_CORS = os.environ.get('ENABLE_CORS') or False + ALLOWED_EXTENSIONS = {'csv'} + MAX_CONTENT_LENGTH = 10 * 1024 * 1024 # 10 MB + SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_DATABASE_URI = f'sqlite:///{BASE_DIR / "db.sqlite"}' diff --git a/backend/app/coordinator/routes/students.py b/backend/app/coordinator/routes/students.py index c5a90b7..db51903 100644 --- a/backend/app/coordinator/routes/students.py +++ b/backend/app/coordinator/routes/students.py @@ -1,8 +1,10 @@ from typing import Tuple from random import randint +from itertools import islice from flask import Blueprint, request, Response from marshmallow import ValidationError +from sqlalchemy.exc import IntegrityError from flask_sqlalchemy import get_debug_queries from ...students.models import Student @@ -10,7 +12,7 @@ from ..schemas import StudentSchema, StudentEditSchema, StudentCreateSchema from ...dependencies import db from ..utils import parse_csv from ..exceptions import InvalidNameOrTypeHeaderException -from ...base.utils import paginate_models +from ...base.utils import paginate_models, is_allowed_extensions bp = Blueprint("students", __name__, url_prefix="/students") @@ -30,7 +32,7 @@ def list_students() -> Tuple[dict, int]: response = paginate_models(page, student_query) if isinstance(response, tuple): return response - print(get_debug_queries()[0]) + # print(get_debug_queries()[0]) return {"students": students_schema.dump(response['items']), "max_pages": response['max_pages']}, 200 @@ -100,15 +102,29 @@ def create_student() -> Tuple[dict, int] or Response: @bp.route("/upload/", methods=["POST"]) def upload_students() -> Tuple[dict, int]: """Maybe in the future move to celery workers""" - if (uploaded_file := request.files.get('file')) is None: + if (uploaded_file := request.files.get('file')) is None or uploaded_file.filename == '': return {"error": "You didn't attach a csv file!"}, 400 - try: - students = parse_csv(uploaded_file) - except InvalidNameOrTypeHeaderException: - return {"error": "Invalid format of csv file!"}, 400 + if uploaded_file and is_allowed_extensions(uploaded_file.filename): + try: + students = parse_csv(uploaded_file) + except InvalidNameOrTypeHeaderException: + return {"error": "Invalid format of csv file!"}, 400 + + try: + while True: + sliced_students = islice(students, 5) + list_of_students = list(sliced_students) + + if len(list_of_students) == 0: + break + db.session.add_all(list_of_students) + db.session.commit() + except IntegrityError as e: + # print(e) + # in the future create sql query checks index and add only these students, which didn't exist in db + return {"error": "These students have already exist!"}, 400 + else: + return {"error": "Invalid extension of file"}, 400 - for student in students: - db.session.add(student) - db.session.commit() return {"message": "Students was created by uploading csv file!"}, 200 diff --git a/backend/app/coordinator/utils.py b/backend/app/coordinator/utils.py index 7a49886..07605a6 100644 --- a/backend/app/coordinator/utils.py +++ b/backend/app/coordinator/utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Generator, Any from random import randint import pandas as pd @@ -7,31 +7,29 @@ from .exceptions import InvalidNameOrTypeHeaderException from ..students.models import Student -def check_columns(df: pd.DataFrame, columns: List[str]) -> bool: +def check_columns(df: pd.DataFrame) -> bool: + headers = set(df.keys().values) + columns = ['first_name', 'last_name', 'index', 'mode'] + if len(headers - set(columns)) != 0: + return False + flag = True col_types = ['object', 'object', 'int', 'int'] - print(df.dtypes['first_name']) + for name, col_type in zip(columns, col_types): - if name not in df and df.dtypes[name].startswith(col_type): + if not str(df.dtypes[name]).startswith(col_type): flag = False break return flag -def parse_csv(file) -> List[Student]: +def parse_csv(file) -> Generator[Student, Any, None]: df = pd.read_csv(file) - columns = ['first_name', 'last_name', 'index', 'mode'] - if not check_columns(df, columns): + if not check_columns(df): raise InvalidNameOrTypeHeaderException - students = [] - for _, item in df.iterrows(): - data = {} - for c in columns: - data[c] = item[c] - data['email'] = f'student{randint(1, 300_000)}@gmail.com' - students.append(Student(**data)) - + students = (Student(**dict(item.items(), email=f'student{randint(1, 300_000)}@gmail.com')) + for _, item in df.iterrows()) return students diff --git a/backend/app/errors.py b/backend/app/errors.py new file mode 100644 index 0000000..7cf9b1d --- /dev/null +++ b/backend/app/errors.py @@ -0,0 +1,7 @@ +from typing import Tuple + +from werkzeug.exceptions import RequestEntityTooLarge + + +def request_entity_too_large(error: RequestEntityTooLarge) -> Tuple[dict, int]: + return {'error': 'File too large!'}, 413