update - route of uploading students list - add additional handling with potential errors
This commit is contained in:
parent
22bd08c312
commit
8ac8e869c1
@ -9,6 +9,7 @@ from .dependencies import db, ma
|
|||||||
from .commands.startapp import startapp
|
from .commands.startapp import startapp
|
||||||
from .utils import import_models
|
from .utils import import_models
|
||||||
from .api import api_bp
|
from .api import api_bp
|
||||||
|
from .errors import request_entity_too_large
|
||||||
|
|
||||||
|
|
||||||
def create_app(config_name: str = None) -> Flask:
|
def create_app(config_name: str = None) -> Flask:
|
||||||
@ -35,4 +36,7 @@ def create_app(config_name: str = None) -> Flask:
|
|||||||
# register commands
|
# register commands
|
||||||
app.cli.add_command(startapp)
|
app.cli.add_command(startapp)
|
||||||
|
|
||||||
|
# register errors
|
||||||
|
app.register_error_handler(413, request_entity_too_large)
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
from typing import TypedDict, Tuple
|
from typing import TypedDict, Tuple
|
||||||
|
|
||||||
|
from flask import current_app
|
||||||
from flask_sqlalchemy import BaseQuery
|
from flask_sqlalchemy import BaseQuery
|
||||||
from sqlalchemy import desc
|
from sqlalchemy import desc
|
||||||
|
|
||||||
@ -36,3 +37,8 @@ def paginate_models(page: str, query: BaseQuery) -> PaginationResponse or Tuple[
|
|||||||
'items': query.items,
|
'items': query.items,
|
||||||
'max_pages': query.pages
|
'max_pages': query.pages
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def is_allowed_extensions(filename: str):
|
||||||
|
return '.' in filename and \
|
||||||
|
filename.rsplit('.', 1)[1].lower() in current_app.config['ALLOWED_EXTENSIONS']
|
||||||
|
@ -8,8 +8,12 @@ class Config:
|
|||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
SRC_DIR = BASE_DIR / "app"
|
SRC_DIR = BASE_DIR / "app"
|
||||||
EXCLUDED_DIRS = ["__pycache__", "commands"]
|
EXCLUDED_DIRS = ["__pycache__", "commands"]
|
||||||
|
|
||||||
ENABLE_CORS = os.environ.get('ENABLE_CORS') or False
|
ENABLE_CORS = os.environ.get('ENABLE_CORS') or False
|
||||||
|
|
||||||
|
ALLOWED_EXTENSIONS = {'csv'}
|
||||||
|
MAX_CONTENT_LENGTH = 10 * 1024 * 1024 # 10 MB
|
||||||
|
|
||||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||||
SQLALCHEMY_DATABASE_URI = f'sqlite:///{BASE_DIR / "db.sqlite"}'
|
SQLALCHEMY_DATABASE_URI = f'sqlite:///{BASE_DIR / "db.sqlite"}'
|
||||||
|
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
from random import randint
|
from random import randint
|
||||||
|
from itertools import islice
|
||||||
|
|
||||||
from flask import Blueprint, request, Response
|
from flask import Blueprint, request, Response
|
||||||
from marshmallow import ValidationError
|
from marshmallow import ValidationError
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from flask_sqlalchemy import get_debug_queries
|
from flask_sqlalchemy import get_debug_queries
|
||||||
|
|
||||||
from ...students.models import Student
|
from ...students.models import Student
|
||||||
@ -10,7 +12,7 @@ from ..schemas import StudentSchema, StudentEditSchema, StudentCreateSchema
|
|||||||
from ...dependencies import db
|
from ...dependencies import db
|
||||||
from ..utils import parse_csv
|
from ..utils import parse_csv
|
||||||
from ..exceptions import InvalidNameOrTypeHeaderException
|
from ..exceptions import InvalidNameOrTypeHeaderException
|
||||||
from ...base.utils import paginate_models
|
from ...base.utils import paginate_models, is_allowed_extensions
|
||||||
|
|
||||||
bp = Blueprint("students", __name__, url_prefix="/students")
|
bp = Blueprint("students", __name__, url_prefix="/students")
|
||||||
|
|
||||||
@ -30,7 +32,7 @@ def list_students() -> Tuple[dict, int]:
|
|||||||
response = paginate_models(page, student_query)
|
response = paginate_models(page, student_query)
|
||||||
if isinstance(response, tuple):
|
if isinstance(response, tuple):
|
||||||
return response
|
return response
|
||||||
print(get_debug_queries()[0])
|
# print(get_debug_queries()[0])
|
||||||
return {"students": students_schema.dump(response['items']), "max_pages": response['max_pages']}, 200
|
return {"students": students_schema.dump(response['items']), "max_pages": response['max_pages']}, 200
|
||||||
|
|
||||||
|
|
||||||
@ -100,15 +102,29 @@ def create_student() -> Tuple[dict, int] or Response:
|
|||||||
@bp.route("/upload/", methods=["POST"])
|
@bp.route("/upload/", methods=["POST"])
|
||||||
def upload_students() -> Tuple[dict, int]:
|
def upload_students() -> Tuple[dict, int]:
|
||||||
"""Maybe in the future move to celery workers"""
|
"""Maybe in the future move to celery workers"""
|
||||||
if (uploaded_file := request.files.get('file')) is None:
|
if (uploaded_file := request.files.get('file')) is None or uploaded_file.filename == '':
|
||||||
return {"error": "You didn't attach a csv file!"}, 400
|
return {"error": "You didn't attach a csv file!"}, 400
|
||||||
|
|
||||||
try:
|
if uploaded_file and is_allowed_extensions(uploaded_file.filename):
|
||||||
students = parse_csv(uploaded_file)
|
try:
|
||||||
except InvalidNameOrTypeHeaderException:
|
students = parse_csv(uploaded_file)
|
||||||
return {"error": "Invalid format of csv file!"}, 400
|
except InvalidNameOrTypeHeaderException:
|
||||||
|
return {"error": "Invalid format of csv file!"}, 400
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
sliced_students = islice(students, 5)
|
||||||
|
list_of_students = list(sliced_students)
|
||||||
|
|
||||||
|
if len(list_of_students) == 0:
|
||||||
|
break
|
||||||
|
db.session.add_all(list_of_students)
|
||||||
|
db.session.commit()
|
||||||
|
except IntegrityError as e:
|
||||||
|
# print(e)
|
||||||
|
# in the future create sql query checks index and add only these students, which didn't exist in db
|
||||||
|
return {"error": "These students have already exist!"}, 400
|
||||||
|
else:
|
||||||
|
return {"error": "Invalid extension of file"}, 400
|
||||||
|
|
||||||
for student in students:
|
|
||||||
db.session.add(student)
|
|
||||||
db.session.commit()
|
|
||||||
return {"message": "Students was created by uploading csv file!"}, 200
|
return {"message": "Students was created by uploading csv file!"}, 200
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
from typing import List
|
from typing import Generator, Any
|
||||||
from random import randint
|
from random import randint
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
@ -7,31 +7,29 @@ from .exceptions import InvalidNameOrTypeHeaderException
|
|||||||
from ..students.models import Student
|
from ..students.models import Student
|
||||||
|
|
||||||
|
|
||||||
def check_columns(df: pd.DataFrame, columns: List[str]) -> bool:
|
def check_columns(df: pd.DataFrame) -> bool:
|
||||||
|
headers = set(df.keys().values)
|
||||||
|
columns = ['first_name', 'last_name', 'index', 'mode']
|
||||||
|
if len(headers - set(columns)) != 0:
|
||||||
|
return False
|
||||||
|
|
||||||
flag = True
|
flag = True
|
||||||
col_types = ['object', 'object', 'int', 'int']
|
col_types = ['object', 'object', 'int', 'int']
|
||||||
print(df.dtypes['first_name'])
|
|
||||||
for name, col_type in zip(columns, col_types):
|
for name, col_type in zip(columns, col_types):
|
||||||
if name not in df and df.dtypes[name].startswith(col_type):
|
if not str(df.dtypes[name]).startswith(col_type):
|
||||||
flag = False
|
flag = False
|
||||||
break
|
break
|
||||||
|
|
||||||
return flag
|
return flag
|
||||||
|
|
||||||
|
|
||||||
def parse_csv(file) -> List[Student]:
|
def parse_csv(file) -> Generator[Student, Any, None]:
|
||||||
df = pd.read_csv(file)
|
df = pd.read_csv(file)
|
||||||
columns = ['first_name', 'last_name', 'index', 'mode']
|
|
||||||
|
|
||||||
if not check_columns(df, columns):
|
if not check_columns(df):
|
||||||
raise InvalidNameOrTypeHeaderException
|
raise InvalidNameOrTypeHeaderException
|
||||||
|
|
||||||
students = []
|
students = (Student(**dict(item.items(), email=f'student{randint(1, 300_000)}@gmail.com'))
|
||||||
for _, item in df.iterrows():
|
for _, item in df.iterrows())
|
||||||
data = {}
|
|
||||||
for c in columns:
|
|
||||||
data[c] = item[c]
|
|
||||||
data['email'] = f'student{randint(1, 300_000)}@gmail.com'
|
|
||||||
students.append(Student(**data))
|
|
||||||
|
|
||||||
return students
|
return students
|
||||||
|
7
backend/app/errors.py
Normal file
7
backend/app/errors.py
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
from typing import Tuple
|
||||||
|
|
||||||
|
from werkzeug.exceptions import RequestEntityTooLarge
|
||||||
|
|
||||||
|
|
||||||
|
def request_entity_too_large(error: RequestEntityTooLarge) -> Tuple[dict, int]:
|
||||||
|
return {'error': 'File too large!'}, 413
|
Loading…
Reference in New Issue
Block a user