diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/api/BookController.java b/src/main/java/pl/edu/amu/wmi/bookapi/api/BookController.java index 257d6dc..cddf839 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/api/BookController.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/api/BookController.java @@ -16,6 +16,8 @@ import org.springframework.web.multipart.MultipartFile; import pl.edu.amu.wmi.bookapi.api.dto.BookDto; import pl.edu.amu.wmi.bookapi.service.BookService; +import javax.validation.Valid; + @RestController @RequestMapping("/api/books") public class BookController { @@ -44,6 +46,8 @@ public class BookController { @PatchMapping("/{bookId}") public ResponseEntity updateBook(@RequestBody BookDto bookDto, @PathVariable String bookId) { String username = getUserName(); + System.out.println("Book dto"); + System.out.println(bookDto); return ResponseEntity.ok( bookService.updateBook(bookId ,username, bookDto) ); @@ -51,6 +55,7 @@ public class BookController { @DeleteMapping("/{bookId}") public ResponseEntity deleteBook(@PathVariable String bookId) { + System.out.println("Deleting book for user " + getUserName() + " of Id " + bookId); bookService.deleteBook( getUserName(), bookId @@ -60,7 +65,7 @@ public class BookController { } @PostMapping - public ResponseEntity addBook(@RequestBody BookDto bookDto) { + public ResponseEntity addBook(@RequestBody @Valid BookDto bookDto) { bookService.saveBook(getUserName(), bookDto); return ResponseEntity.ok().build(); } @@ -75,8 +80,6 @@ public class BookController { } private String getUserName() { - String username = SecurityContextHolder.getContext().getAuthentication().getPrincipal().toString(); - System.out.println("Username - " + username); - return username; + return SecurityContextHolder.getContext().getAuthentication().getPrincipal().toString(); } } diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/api/dto/BookDto.java b/src/main/java/pl/edu/amu/wmi/bookapi/api/dto/BookDto.java index 265f026..7835543 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/api/dto/BookDto.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/api/dto/BookDto.java @@ -1,8 +1,13 @@ package pl.edu.amu.wmi.bookapi.api.dto; +import javax.validation.constraints.NotNull; + public class BookDto { + @NotNull private String ean; + @NotNull private String author; + @NotNull private String title; public BookDto(String ean, String author, String title) { @@ -34,4 +39,13 @@ public class BookDto { public void setTitle(String title) { this.title = title; } + + @Override + public String toString() { + return "BookDto{" + + "ean='" + ean + '\'' + + ", author='" + author + '\'' + + ", title='" + title + '\'' + + '}'; + } } diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/config/ImageProcessingConfigurationProperties.java b/src/main/java/pl/edu/amu/wmi/bookapi/config/ImageProcessingConfigurationProperties.java new file mode 100644 index 0000000..06bb1e4 --- /dev/null +++ b/src/main/java/pl/edu/amu/wmi/bookapi/config/ImageProcessingConfigurationProperties.java @@ -0,0 +1,25 @@ +package pl.edu.amu.wmi.bookapi.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "imgproc") +public class ImageProcessingConfigurationProperties { + private Boolean debug; + + public Boolean getDebug() { + return debug; + } + + public void setDebug(Boolean debug) { + this.debug = debug; + } + + @Override + public String toString() { + return "ImageProcessingConfigurationProperties{" + + "debug=" + debug + + '}'; + } +} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/EanDetectedButUnableToReadException.java b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/EanDetectedButUnableToReadException.java new file mode 100644 index 0000000..d9853d4 --- /dev/null +++ b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/EanDetectedButUnableToReadException.java @@ -0,0 +1,4 @@ +package pl.edu.amu.wmi.bookapi.exceptions; + +public class EanDetectedButUnableToReadException extends RuntimeException { +} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/ExceptionHandlerFilter.java b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/ExceptionHandlerFilter.java index 2ee6ccc..fcd6b07 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/ExceptionHandlerFilter.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/ExceptionHandlerFilter.java @@ -12,4 +12,16 @@ public class ExceptionHandlerFilter { .status(HttpStatus.BAD_REQUEST) .body("Cannot use this username"); } + + @ExceptionHandler(NoEanCodeDetectedException.class) public ResponseEntity handleNoEan() { + return ResponseEntity + .status(HttpStatus.UNPROCESSABLE_ENTITY) + .body("Item created but no code detected - opencv"); + } + + @ExceptionHandler(EanDetectedButUnableToReadException.class) public ResponseEntity handleEanReadingError() { + return ResponseEntity + .status(HttpStatus.UNPROCESSABLE_ENTITY) + .body("Item created, code detected but error during read - zxing"); + } } diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/NoEanCodeDetectedException.java b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/NoEanCodeDetectedException.java new file mode 100644 index 0000000..3470a5e --- /dev/null +++ b/src/main/java/pl/edu/amu/wmi/bookapi/exceptions/NoEanCodeDetectedException.java @@ -0,0 +1,6 @@ +package pl.edu.amu.wmi.bookapi.exceptions; + +public class NoEanCodeDetectedException extends RuntimeException { + public NoEanCodeDetectedException() { + } +} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepository.java b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepository.java new file mode 100644 index 0000000..3af5e4a --- /dev/null +++ b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepository.java @@ -0,0 +1,9 @@ +package pl.edu.amu.wmi.bookapi.repositories; + +import pl.edu.amu.wmi.bookapi.models.MessageDocument; + +import java.util.List; + +public interface MessageCustomRepository { + List findByUserAndThreadId(String threadId, String user); +} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepositoryImpl.java b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepositoryImpl.java new file mode 100644 index 0000000..d44f501 --- /dev/null +++ b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepositoryImpl.java @@ -0,0 +1,31 @@ +package pl.edu.amu.wmi.bookapi.repositories; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import pl.edu.amu.wmi.bookapi.models.MessageDocument; + +import java.util.List; + +public class MessageCustomRepositoryImpl implements MessageCustomRepository { + + @Autowired + MongoTemplate mongoTemplate; + + @Override + public List findByUserAndThreadId(String threadId, String user) { + Query query = new Query( + Criteria.where("threadId").is(threadId) + .orOperator( + Criteria.where("author").is(user), + Criteria.where("recipient").is(user) + ) + ); + + Sort sort = Sort.by(Sort.Direction.DESC, "createdAt"); + + return mongoTemplate.find(query.with(sort), MessageDocument.class); + } +} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageRepository.java b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageRepository.java index 64f81f3..a8a44d3 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageRepository.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageRepository.java @@ -1,12 +1,6 @@ package pl.edu.amu.wmi.bookapi.repositories; import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; import pl.edu.amu.wmi.bookapi.models.MessageDocument; -import java.util.List; - -public interface MessageRepository extends MongoRepository { - @Query("{ $and: [$or: [{ 'recipient' : ?1}, { 'author': ?1}], {'threadId' : ?0} ]}") - List findByUserAndThreadId(String threadId, String user); -} +public interface MessageRepository extends MongoRepository, MessageCustomRepository {} diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/service/BookService.java b/src/main/java/pl/edu/amu/wmi/bookapi/service/BookService.java index eb029d2..697b9e5 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/service/BookService.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/service/BookService.java @@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import pl.edu.amu.wmi.bookapi.api.dto.BookDto; +import pl.edu.amu.wmi.bookapi.exceptions.NoEanCodeDetectedException; import pl.edu.amu.wmi.bookapi.models.BookDocument; import pl.edu.amu.wmi.bookapi.repositories.BookRepository; @@ -30,14 +31,15 @@ public class BookService { } public void saveBook(String userName, BookDto bookDto) { - System.out.println("saving"); - System.out.println(bookRepository.save(BookDocument.from(userName, bookDto))); + bookRepository.save(BookDocument.from(userName, bookDto)); } public void handleImageUpload(MultipartFile imageFile, String author, String title) throws Exception { String detectedEan = imageProcessingService.getDecodedEan(imageFile); - if (detectedEan == null) detectedEan = "Test"; - saveBook("admin", new BookDto(detectedEan, author, title)); + saveBook(author, new BookDto(detectedEan, author, title)); + if (detectedEan.isEmpty()) { + throw new NoEanCodeDetectedException(); + } } public List findAll() { diff --git a/src/main/java/pl/edu/amu/wmi/bookapi/service/ImageProcessingService.java b/src/main/java/pl/edu/amu/wmi/bookapi/service/ImageProcessingService.java index f9c96d9..27cab11 100644 --- a/src/main/java/pl/edu/amu/wmi/bookapi/service/ImageProcessingService.java +++ b/src/main/java/pl/edu/amu/wmi/bookapi/service/ImageProcessingService.java @@ -15,47 +15,70 @@ import org.bytedeco.opencv.global.opencv_imgproc; import org.bytedeco.opencv.opencv_core.Mat; import org.bytedeco.opencv.opencv_core.MatVector; import org.bytedeco.opencv.opencv_core.Size; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import pl.edu.amu.wmi.bookapi.config.ImageProcessingConfigurationProperties; +import pl.edu.amu.wmi.bookapi.exceptions.EanDetectedButUnableToReadException; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.awt.image.DataBuffer; -import java.awt.image.DataBufferByte; -import java.awt.image.DataBufferInt; import java.io.File; import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; import static org.bytedeco.opencv.global.opencv_core.CV_32F; -import static org.bytedeco.opencv.global.opencv_core.CV_8UC3; import static org.bytedeco.opencv.global.opencv_core.convertScaleAbs; import static org.bytedeco.opencv.global.opencv_core.subtract; +import static org.bytedeco.opencv.global.opencv_imgcodecs.imread; import static org.bytedeco.opencv.global.opencv_imgcodecs.imwrite; -import static org.bytedeco.opencv.global.opencv_imgproc.CHAIN_APPROX_SIMPLE; -import static org.bytedeco.opencv.global.opencv_imgproc.COLOR_BGR2GRAY; -import static org.bytedeco.opencv.global.opencv_imgproc.MORPH_CLOSE; -import static org.bytedeco.opencv.global.opencv_imgproc.MORPH_RECT; -import static org.bytedeco.opencv.global.opencv_imgproc.RETR_EXTERNAL; -import static org.bytedeco.opencv.global.opencv_imgproc.Sobel; -import static org.bytedeco.opencv.global.opencv_imgproc.THRESH_BINARY; +import static org.bytedeco.opencv.global.opencv_imgproc.Scharr; import static org.bytedeco.opencv.global.opencv_imgproc.blur; +import static org.bytedeco.opencv.global.opencv_imgproc.contourArea; import static org.bytedeco.opencv.global.opencv_imgproc.cvtColor; import static org.bytedeco.opencv.global.opencv_imgproc.findContours; import static org.bytedeco.opencv.global.opencv_imgproc.getStructuringElement; import static org.bytedeco.opencv.global.opencv_imgproc.morphologyEx; import static org.bytedeco.opencv.global.opencv_imgproc.threshold; +import static org.opencv.imgproc.Imgproc.CHAIN_APPROX_SIMPLE; +import static org.opencv.imgproc.Imgproc.COLOR_BGR2GRAY; +import static org.opencv.imgproc.Imgproc.MORPH_CLOSE; +import static org.opencv.imgproc.Imgproc.MORPH_RECT; +import static org.opencv.imgproc.Imgproc.RETR_EXTERNAL; +import static org.opencv.imgproc.Imgproc.THRESH_BINARY; @Service public class ImageProcessingService { - public ImageProcessingService() { + private Logger log = LoggerFactory.getLogger(ImageProcessingService.class); + private ImageProcessingConfigurationProperties imageProcessingConfigurationProperties; + private static final Map contentTypeConverter = Map.of( + "image/jpeg", "jpg", + "image/png", "png"); + + private class InvalidTypeException extends RuntimeException { + public InvalidTypeException(String message) { + super(message); + } + } + + @Autowired + public ImageProcessingService(ImageProcessingConfigurationProperties imageProcessingConfigurationProperties) { + this.imageProcessingConfigurationProperties = imageProcessingConfigurationProperties; } public String getDecodedEan(MultipartFile imageFile) throws Exception { - checkIfImageContainsBarcode(imageFile); - - return getBarcode(imageFile); + if (doesImageContainBarcode(imageFile)){ + return getBarcode(imageFile); + } + return ""; } private String getBarcode(MultipartFile imageFile) throws IOException { @@ -68,125 +91,127 @@ public class ImageProcessingService { return result.getText(); } catch (NotFoundException | FormatException | ChecksumException e) { e.printStackTrace(); - return null; + throw new EanDetectedButUnableToReadException(); } } - private void checkIfImageContainsBarcode(MultipartFile imageFile) throws IOException{ - BufferedImage img = ImageIO.read(imageFile.getInputStream()); - File outputfile = new File("start.jpg"); - ImageIO.write(img, "jpg", outputfile); + private Boolean doesImageContainBarcode(MultipartFile imageFile) throws IOException{ + String uid = UUID.randomUUID().toString(); + String currentPath = Paths.get(".").toAbsolutePath().normalize().toString(); + String rawContentType = imageFile.getContentType(); + + if (!contentTypeConverter.containsKey(rawContentType)) { throw new InvalidTypeException("Only png and jpeg files accepted"); } + + BufferedImage img = ImageIO.read(imageFile.getInputStream()); + String contentType = contentTypeConverter.get(rawContentType); + + String path = currentPath + "/" + uid + "." + contentType; + File outputfile = new File(path); + ImageIO.write(img, contentType, outputfile); + + Mat src = imread(path); + saveDebugImage(src, uid, "initial", contentType); - Mat src = matify(img); Mat dst = new Mat(); cvtColor(src, dst, COLOR_BGR2GRAY); - imwrite("./before.png", src); - imwrite("./test.png", dst); - Mat sobel = new Mat(); + saveDebugImage(dst, uid, "bgr2gray", contentType); Mat matScharX = new Mat(); Mat matScharY = new Mat(); -// Scharr(dst, matScharX, CV_32F, 1, 0, 1.0d, -1.0, 0); -// Scharr(dst, matScharX, CV_32F, 0, 1, 1.0d, -1.0, 0); - Sobel(dst, matScharX, CV_32F, 1, 0, -1, 1.0, 1.0, 1); - Sobel(dst, matScharY, CV_32F, 0, 1, -1, 1.0, 1.0, 1); + Scharr(dst, matScharX, CV_32F, 1, 0, 1.0d, -1.0, 0); + Scharr(dst, matScharY, CV_32F, 0, 1, 1.0d, -1.0, 0); subtract(matScharX, matScharY); + Mat conv = new Mat(); convertScaleAbs(matScharX, conv); - imwrite("./scharr.png", conv); + + saveDebugImage(conv, uid, "scharr", contentType); Mat blurDestination = new Mat(); blur(conv, blurDestination, new Size(9,9)); - Mat thresholdDestination = new Mat(); - threshold(blurDestination, thresholdDestination, 230d, 255d, THRESH_BINARY); + saveDebugImage(blurDestination, uid, "blur", contentType); - imwrite("./threshold.png", thresholdDestination); + Mat thresholdDestination = new Mat(); + threshold(blurDestination, thresholdDestination, 200d, 255d, THRESH_BINARY); + + saveDebugImage(thresholdDestination, uid, "threshold", contentType); Mat kernel = getStructuringElement(MORPH_RECT, new Size(21, 7)); Mat morph = new Mat(); morphologyEx(thresholdDestination, morph, MORPH_CLOSE, kernel); - imwrite("./morph.png", morph); -// -// Mat erode = new Mat(); -// imwrite("./erodeBefore.png", morph); -// opencv_imgproc.erode(morph, erode, new Mat()); -// for(int i =0 ; i < 3 ; i++) { -// opencv_imgproc.erode(erode, erode, new Mat()); -// } -// -// Mat dilate = new Mat(); -// opencv_imgproc.dilate(morph, dilate, new Mat()); -// for(int i =0 ; i < 3 ; i++) { -// opencv_imgproc.erode(dilate, dilate, new Mat()); -// } + saveDebugImage(morph, uid, "morph", contentType); - imwrite("./erodeAfter.png", morph); - MatVector contour = new MatVector(); - findContours(morph, contour ,RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); - int count = 0; - for (int i = 0 ; i < contour.size(); i++){ - double minContourArea = (double) img.getHeight() * img.getWidth() * 0.05; - System.out.println("Min contour " + minContourArea + "current contour " + opencv_imgproc.contourArea(contour.get(i))); - if(opencv_imgproc.contourArea(contour.get(i)) > minContourArea) { - System.out.println("Contour " + i); - System.out.println(opencv_imgproc.contourArea(contour.get(i))); - count++; - } + opencv_imgproc.erode(morph, morph, new Mat()); + for(int i = 0 ; i < 2 ; i++) { + opencv_imgproc.erode(conv, conv, new Mat()); } - System.out.println("No of contours higher than threshold"); - System.out.println(count); - System.out.println("Total contours"); - System.out.println(contour.size()); + saveDebugImage(morph, uid, "erode", contentType); + + for(int i = 0 ; i < 2 ; i++) { + opencv_imgproc.dilate(morph, morph, new Mat()); + } + + saveDebugImage(morph, uid, "dilate", contentType); + + MatVector contours = new MatVector(); + findContours(morph, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + System.out.println("Found " + contours.size() + " contours"); + + saveDebugImage(morph, uid, "final", contentType); + + Double totalQualifyingContours = getQualifyingContoursAreas( + contours, + img + ).stream().reduce(0.0, Double::sum); + + System.out.println("Total qualifying contours " + totalQualifyingContours); + System.out.println("Total area " + img.getWidth() * img.getHeight()); + System.out.println("Minimum area " + img.getWidth() * img.getHeight() * 0.05); + return totalQualifyingContours > img.getWidth() * img.getHeight() * 0.05; } - public Mat matify(BufferedImage sourceImg) { - System.out.println("Matify"); - long millis = System.currentTimeMillis(); - - DataBuffer dataBuffer = sourceImg.getRaster().getDataBuffer(); - byte[] imgPixels = null; - Mat imgMat = null; - - int width = sourceImg.getWidth(); - int height = sourceImg.getHeight(); - - if(dataBuffer instanceof DataBufferByte) { - imgPixels = ((DataBufferByte)dataBuffer).getData(); - } - - if(dataBuffer instanceof DataBufferInt) { - System.out.println("Matify1"); - - int byteSize = width * height; - imgPixels = new byte[byteSize*3]; - - int[] imgIntegerPixels = ((DataBufferInt)dataBuffer).getData(); - System.out.println("Matify2"); - - for(int p = 0; p < byteSize; p++) { - imgPixels[p*3 + 0] = (byte) ((imgIntegerPixels[p] & 0x00FF0000) >> 16); - imgPixels[p*3 + 1] = (byte) ((imgIntegerPixels[p] & 0x0000FF00) >> 8); - imgPixels[p*3 + 2] = (byte) (imgIntegerPixels[p] & 0x000000FF); + private List getQualifyingContoursAreas(MatVector contours, BufferedImage image) { + List contoursAreas = new ArrayList<>(); + System.out.println("Starting contour evaluation, total " + contours.size()); + for(int i = 0 ; i < contours.size() ; i++) { + Double area = contourArea(contours.get(i)); + if(isContourQualifiable( + contourArea(contours.get(i)), + (double) image.getWidth() * image.getHeight() + )) { + System.out.println("Contour #" + + i + + " is qualified by area " + + contourArea(contours.get(i)) + + " - minimum " + + image.getHeight() * image.getWidth() * 0.01); + contoursAreas.add(area); + } else { + System.out.println("Contour #" + + i + + " is NOT qualified by area " + + contourArea(contours.get(i)) + + " - minimum " + + image.getHeight() * image.getWidth() * 0.01); } - System.out.println("Matify"); - } - if(imgPixels != null) { - imgMat = new Mat(height, width, CV_8UC3); - imgMat.data().put(imgPixels); + return contoursAreas; + } + + private Boolean isContourQualifiable(Double contourArea, Double imageSize) { + return contourArea > imageSize * 0.01; + } + + private void saveDebugImage(Mat imgToWrite, String uid, String operationName, String contentType) { + if(imageProcessingConfigurationProperties.getDebug()) { + imwrite("./" + uid + "__" + operationName + "." + contentType, imgToWrite); } - - Mat mat = new Mat(); - - System.out.println("matify exec millis: " + (System.currentTimeMillis() - millis)); - - return imgMat; } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8b13789..4ea2219 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1 +1 @@ - +imgproc.debug = true