s449314
/
opencv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored

This commit is contained in:
Artur Kmieckowiak 2020-02-23 17:10:06 +01:00
parent 14a26730cd
commit c3c4b1202f
12 changed files with 242 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/pl/edu/amu/wmi/bookapi/api/BookController.java
View File

@ -16,6 +16,8 @@ import org.springframework.web.multipart.MultipartFile;
import pl.edu.amu.wmi.bookapi.api.dto.BookDto;
import pl.edu.amu.wmi.bookapi.service.BookService;
import javax.validation.Valid;
@RestController
@RequestMapping("/api/books")
public class BookController {
@ -44,6 +46,8 @@ public class BookController {
@PatchMapping("/{bookId}")
public ResponseEntity updateBook(@RequestBody BookDto bookDto, @PathVariable String bookId) {
String username = getUserName();
System.out.println("Book dto");
System.out.println(bookDto);
return ResponseEntity.ok(
bookService.updateBook(bookId ,username, bookDto)
);
@ -51,6 +55,7 @@ public class BookController {
@DeleteMapping("/{bookId}")
public ResponseEntity deleteBook(@PathVariable String bookId) {
System.out.println("Deleting book for user " + getUserName() + " of Id " + bookId);
bookService.deleteBook(
getUserName(),
bookId
@ -60,7 +65,7 @@ public class BookController {
}
@PostMapping
public ResponseEntity addBook(@RequestBody BookDto bookDto) {
public ResponseEntity addBook(@RequestBody @Valid BookDto bookDto) {
bookService.saveBook(getUserName(), bookDto);
return ResponseEntity.ok().build();
}
@ -75,8 +80,6 @@ public class BookController {
}
private String getUserName() {
String username = SecurityContextHolder.getContext().getAuthentication().getPrincipal().toString();
System.out.println("Username - " + username);
return username;
return SecurityContextHolder.getContext().getAuthentication().getPrincipal().toString();
}
}

14
src/main/java/pl/edu/amu/wmi/bookapi/api/dto/BookDto.java
View File

@ -1,8 +1,13 @@
package pl.edu.amu.wmi.bookapi.api.dto;
import javax.validation.constraints.NotNull;
public class BookDto {
@NotNull
private String ean;
@NotNull
private String author;
@NotNull
private String title;
public BookDto(String ean, String author, String title) {
@ -34,4 +39,13 @@ public class BookDto {
public void setTitle(String title) {
this.title = title;
}
@Override
public String toString() {
return "BookDto{" +
"ean='" + ean + '\'' +
", author='" + author + '\'' +
", title='" + title + '\'' +
'}';
}
}

25
src/main/java/pl/edu/amu/wmi/bookapi/config/ImageProcessingConfigurationProperties.java Normal file
View File

@ -0,0 +1,25 @@
package pl.edu.amu.wmi.bookapi.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties(prefix = "imgproc")
public class ImageProcessingConfigurationProperties {
private Boolean debug;
public Boolean getDebug() {
return debug;
}
public void setDebug(Boolean debug) {
this.debug = debug;
}
@Override
public String toString() {
return "ImageProcessingConfigurationProperties{" +
"debug=" + debug +
'}';
}
}

4
src/main/java/pl/edu/amu/wmi/bookapi/exceptions/EanDetectedButUnableToReadException.java Normal file
View File

@ -0,0 +1,4 @@
package pl.edu.amu.wmi.bookapi.exceptions;
public class EanDetectedButUnableToReadException extends RuntimeException {
}

12
src/main/java/pl/edu/amu/wmi/bookapi/exceptions/ExceptionHandlerFilter.java
View File

@ -12,4 +12,16 @@ public class ExceptionHandlerFilter {
.status(HttpStatus.BAD_REQUEST)
.body("Cannot use this username");
}
@ExceptionHandler(NoEanCodeDetectedException.class) public ResponseEntity handleNoEan() {
return ResponseEntity
.status(HttpStatus.UNPROCESSABLE_ENTITY)
.body("Item created but no code detected - opencv");
}
@ExceptionHandler(EanDetectedButUnableToReadException.class) public ResponseEntity handleEanReadingError() {
return ResponseEntity
.status(HttpStatus.UNPROCESSABLE_ENTITY)
.body("Item created, code detected but error during read - zxing");
}
}

6
src/main/java/pl/edu/amu/wmi/bookapi/exceptions/NoEanCodeDetectedException.java Normal file
View File

@ -0,0 +1,6 @@
package pl.edu.amu.wmi.bookapi.exceptions;
public class NoEanCodeDetectedException extends RuntimeException {
public NoEanCodeDetectedException() {
}
}

9
src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepository.java Normal file
View File

@ -0,0 +1,9 @@
package pl.edu.amu.wmi.bookapi.repositories;
import pl.edu.amu.wmi.bookapi.models.MessageDocument;
import java.util.List;
public interface MessageCustomRepository {
List<MessageDocument> findByUserAndThreadId(String threadId, String user);
}

31
src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageCustomRepositoryImpl.java Normal file
View File

@ -0,0 +1,31 @@
package pl.edu.amu.wmi.bookapi.repositories;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import pl.edu.amu.wmi.bookapi.models.MessageDocument;
import java.util.List;
public class MessageCustomRepositoryImpl implements MessageCustomRepository {
@Autowired
MongoTemplate mongoTemplate;
@Override
public List<MessageDocument> findByUserAndThreadId(String threadId, String user) {
Query query = new Query(
Criteria.where("threadId").is(threadId)
.orOperator(
Criteria.where("author").is(user),
Criteria.where("recipient").is(user)
)
);
Sort sort = Sort.by(Sort.Direction.DESC, "createdAt");
return mongoTemplate.find(query.with(sort), MessageDocument.class);
}
}

8
src/main/java/pl/edu/amu/wmi/bookapi/repositories/MessageRepository.java
View File

@ -1,12 +1,6 @@
package pl.edu.amu.wmi.bookapi.repositories;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import pl.edu.amu.wmi.bookapi.models.MessageDocument;
import java.util.List;
public interface MessageRepository extends MongoRepository<MessageDocument, String> {
@Query("{ $and: [$or: [{ 'recipient' : ?1}, { 'author': ?1}], {'threadId' : ?0} ]}")
List<MessageDocument> findByUserAndThreadId(String threadId, String user);
}
public interface MessageRepository extends MongoRepository<MessageDocument, String>, MessageCustomRepository {}

10
src/main/java/pl/edu/amu/wmi/bookapi/service/BookService.java
View File

@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import pl.edu.amu.wmi.bookapi.api.dto.BookDto;
import pl.edu.amu.wmi.bookapi.exceptions.NoEanCodeDetectedException;
import pl.edu.amu.wmi.bookapi.models.BookDocument;
import pl.edu.amu.wmi.bookapi.repositories.BookRepository;
@ -30,14 +31,15 @@ public class BookService {
}
public void saveBook(String userName, BookDto bookDto) {
System.out.println("saving");
System.out.println(bookRepository.save(BookDocument.from(userName, bookDto)));
bookRepository.save(BookDocument.from(userName, bookDto));
}
public void handleImageUpload(MultipartFile imageFile, String author, String title) throws Exception {
String detectedEan = imageProcessingService.getDecodedEan(imageFile);
if (detectedEan == null) detectedEan = "Test";
saveBook("admin", new BookDto(detectedEan, author, title));
saveBook(author, new BookDto(detectedEan, author, title));
if (detectedEan.isEmpty()) {
throw new NoEanCodeDetectedException();
}
}
public List<BookDocument> findAll() {

227
src/main/java/pl/edu/amu/wmi/bookapi/service/ImageProcessingService.java
View File

@ -15,47 +15,70 @@ import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.MatVector;
import org.bytedeco.opencv.opencv_core.Size;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import pl.edu.amu.wmi.bookapi.config.ImageProcessingConfigurationProperties;
import pl.edu.amu.wmi.bookapi.exceptions.EanDetectedButUnableToReadException;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.DataBufferInt;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.bytedeco.opencv.global.opencv_core.CV_32F;
import static org.bytedeco.opencv.global.opencv_core.CV_8UC3;
import static org.bytedeco.opencv.global.opencv_core.convertScaleAbs;
import static org.bytedeco.opencv.global.opencv_core.subtract;
import static org.bytedeco.opencv.global.opencv_imgcodecs.imread;
import static org.bytedeco.opencv.global.opencv_imgcodecs.imwrite;
import static org.bytedeco.opencv.global.opencv_imgproc.CHAIN_APPROX_SIMPLE;
import static org.bytedeco.opencv.global.opencv_imgproc.COLOR_BGR2GRAY;
import static org.bytedeco.opencv.global.opencv_imgproc.MORPH_CLOSE;
import static org.bytedeco.opencv.global.opencv_imgproc.MORPH_RECT;
import static org.bytedeco.opencv.global.opencv_imgproc.RETR_EXTERNAL;
import static org.bytedeco.opencv.global.opencv_imgproc.Sobel;
import static org.bytedeco.opencv.global.opencv_imgproc.THRESH_BINARY;
import static org.bytedeco.opencv.global.opencv_imgproc.Scharr;
import static org.bytedeco.opencv.global.opencv_imgproc.blur;
import static org.bytedeco.opencv.global.opencv_imgproc.contourArea;
import static org.bytedeco.opencv.global.opencv_imgproc.cvtColor;
import static org.bytedeco.opencv.global.opencv_imgproc.findContours;
import static org.bytedeco.opencv.global.opencv_imgproc.getStructuringElement;
import static org.bytedeco.opencv.global.opencv_imgproc.morphologyEx;
import static org.bytedeco.opencv.global.opencv_imgproc.threshold;
import static org.opencv.imgproc.Imgproc.CHAIN_APPROX_SIMPLE;
import static org.opencv.imgproc.Imgproc.COLOR_BGR2GRAY;
import static org.opencv.imgproc.Imgproc.MORPH_CLOSE;
import static org.opencv.imgproc.Imgproc.MORPH_RECT;
import static org.opencv.imgproc.Imgproc.RETR_EXTERNAL;
import static org.opencv.imgproc.Imgproc.THRESH_BINARY;
@Service
public class ImageProcessingService {
public ImageProcessingService() {
private Logger log = LoggerFactory.getLogger(ImageProcessingService.class);
private ImageProcessingConfigurationProperties imageProcessingConfigurationProperties;
private static final Map<String, String> contentTypeConverter = Map.of(
"image/jpeg", "jpg",
"image/png", "png");
private class InvalidTypeException extends RuntimeException {
public InvalidTypeException(String message) {
super(message);
}
}
@Autowired
public ImageProcessingService(ImageProcessingConfigurationProperties imageProcessingConfigurationProperties) {
this.imageProcessingConfigurationProperties = imageProcessingConfigurationProperties;
}
public String getDecodedEan(MultipartFile imageFile) throws Exception {
checkIfImageContainsBarcode(imageFile);
return getBarcode(imageFile);
if (doesImageContainBarcode(imageFile)){
return getBarcode(imageFile);
}
return "";
}
private String getBarcode(MultipartFile imageFile) throws IOException {
@ -68,125 +91,127 @@ public class ImageProcessingService {
return result.getText();
} catch (NotFoundException | FormatException | ChecksumException e) {
e.printStackTrace();
return null;
throw new EanDetectedButUnableToReadException();
}
}
private void checkIfImageContainsBarcode(MultipartFile imageFile) throws IOException{
BufferedImage img = ImageIO.read(imageFile.getInputStream());
File outputfile = new File("start.jpg");
ImageIO.write(img, "jpg", outputfile);
private Boolean doesImageContainBarcode(MultipartFile imageFile) throws IOException{
String uid = UUID.randomUUID().toString();
String currentPath = Paths.get(".").toAbsolutePath().normalize().toString();
String rawContentType = imageFile.getContentType();
if (!contentTypeConverter.containsKey(rawContentType)) { throw new InvalidTypeException("Only png and jpeg files accepted"); }
BufferedImage img = ImageIO.read(imageFile.getInputStream());
String contentType = contentTypeConverter.get(rawContentType);
String path = currentPath + "/" + uid + "." + contentType;
File outputfile = new File(path);
ImageIO.write(img, contentType, outputfile);
Mat src = imread(path);
saveDebugImage(src, uid, "initial", contentType);
Mat src = matify(img);
Mat dst = new Mat();
cvtColor(src, dst, COLOR_BGR2GRAY);
imwrite("./before.png", src);
imwrite("./test.png", dst);
Mat sobel = new Mat();
saveDebugImage(dst, uid, "bgr2gray", contentType);
Mat matScharX = new Mat();
Mat matScharY = new Mat();
// Scharr(dst, matScharX, CV_32F, 1, 0, 1.0d, -1.0, 0);
// Scharr(dst, matScharX, CV_32F, 0, 1, 1.0d, -1.0, 0);
Sobel(dst, matScharX, CV_32F, 1, 0, -1, 1.0, 1.0, 1);
Sobel(dst, matScharY, CV_32F, 0, 1, -1, 1.0, 1.0, 1);
Scharr(dst, matScharX, CV_32F, 1, 0, 1.0d, -1.0, 0);
Scharr(dst, matScharY, CV_32F, 0, 1, 1.0d, -1.0, 0);
subtract(matScharX, matScharY);
Mat conv = new Mat();
convertScaleAbs(matScharX, conv);
imwrite("./scharr.png", conv);
saveDebugImage(conv, uid, "scharr", contentType);
Mat blurDestination = new Mat();
blur(conv, blurDestination, new Size(9,9));
Mat thresholdDestination = new Mat();
threshold(blurDestination, thresholdDestination, 230d, 255d, THRESH_BINARY);
saveDebugImage(blurDestination, uid, "blur", contentType);
imwrite("./threshold.png", thresholdDestination);
Mat thresholdDestination = new Mat();
threshold(blurDestination, thresholdDestination, 200d, 255d, THRESH_BINARY);
saveDebugImage(thresholdDestination, uid, "threshold", contentType);
Mat kernel = getStructuringElement(MORPH_RECT, new Size(21, 7));
Mat morph = new Mat();
morphologyEx(thresholdDestination, morph, MORPH_CLOSE, kernel);
imwrite("./morph.png", morph);
//
// Mat erode = new Mat();
// imwrite("./erodeBefore.png", morph);
// opencv_imgproc.erode(morph, erode, new Mat());
// for(int i =0 ; i < 3 ; i++) {
// opencv_imgproc.erode(erode, erode, new Mat());
// }
//
// Mat dilate = new Mat();
// opencv_imgproc.dilate(morph, dilate, new Mat());
// for(int i =0 ; i < 3 ; i++) {
// opencv_imgproc.erode(dilate, dilate, new Mat());
// }
saveDebugImage(morph, uid, "morph", contentType);
imwrite("./erodeAfter.png", morph);
MatVector contour = new MatVector();
findContours(morph, contour ,RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
int count = 0;
for (int i = 0 ; i < contour.size(); i++){
double minContourArea = (double) img.getHeight() * img.getWidth() * 0.05;
System.out.println("Min contour " + minContourArea + "current contour " + opencv_imgproc.contourArea(contour.get(i)));
if(opencv_imgproc.contourArea(contour.get(i)) > minContourArea) {
System.out.println("Contour " + i);
System.out.println(opencv_imgproc.contourArea(contour.get(i)));
count++;
}
opencv_imgproc.erode(morph, morph, new Mat());
for(int i = 0 ; i < 2 ; i++) {
opencv_imgproc.erode(conv, conv, new Mat());
}
System.out.println("No of contours higher than threshold");
System.out.println(count);
System.out.println("Total contours");
System.out.println(contour.size());
saveDebugImage(morph, uid, "erode", contentType);
for(int i = 0 ; i < 2 ; i++) {
opencv_imgproc.dilate(morph, morph, new Mat());
}
saveDebugImage(morph, uid, "dilate", contentType);
MatVector contours = new MatVector();
findContours(morph, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
System.out.println("Found " + contours.size() + " contours");
saveDebugImage(morph, uid, "final", contentType);
Double totalQualifyingContours = getQualifyingContoursAreas(
contours,
img
).stream().reduce(0.0, Double::sum);
System.out.println("Total qualifying contours " + totalQualifyingContours);
System.out.println("Total area " + img.getWidth() * img.getHeight());
System.out.println("Minimum area " + img.getWidth() * img.getHeight() * 0.05);
return totalQualifyingContours > img.getWidth() * img.getHeight() * 0.05;
}
public Mat matify(BufferedImage sourceImg) {
System.out.println("Matify");
long millis = System.currentTimeMillis();
DataBuffer dataBuffer = sourceImg.getRaster().getDataBuffer();
byte[] imgPixels = null;
Mat imgMat = null;
int width = sourceImg.getWidth();
int height = sourceImg.getHeight();
if(dataBuffer instanceof DataBufferByte) {
imgPixels = ((DataBufferByte)dataBuffer).getData();
}
if(dataBuffer instanceof DataBufferInt) {
System.out.println("Matify1");
int byteSize = width * height;
imgPixels = new byte[byteSize*3];
int[] imgIntegerPixels = ((DataBufferInt)dataBuffer).getData();
System.out.println("Matify2");
for(int p = 0; p < byteSize; p++) {
imgPixels[p*3 + 0] = (byte) ((imgIntegerPixels[p] & 0x00FF0000) >> 16);
imgPixels[p*3 + 1] = (byte) ((imgIntegerPixels[p] & 0x0000FF00) >> 8);
imgPixels[p*3 + 2] = (byte) (imgIntegerPixels[p] & 0x000000FF);
private List<Double> getQualifyingContoursAreas(MatVector contours, BufferedImage image) {
List<Double> contoursAreas = new ArrayList<>();
System.out.println("Starting contour evaluation, total " + contours.size());
for(int i = 0 ; i < contours.size() ; i++) {
Double area = contourArea(contours.get(i));
if(isContourQualifiable(
contourArea(contours.get(i)),
(double) image.getWidth() * image.getHeight()
)) {
System.out.println("Contour #" +
i +
" is qualified by area " +
contourArea(contours.get(i)) +
" - minimum " +
image.getHeight() * image.getWidth() * 0.01);
contoursAreas.add(area);
} else {
System.out.println("Contour #" +
i +
" is NOT qualified by area " +
contourArea(contours.get(i)) +
" - minimum " +
image.getHeight() * image.getWidth() * 0.01);
}
System.out.println("Matify");
}
if(imgPixels != null) {
imgMat = new Mat(height, width, CV_8UC3);
imgMat.data().put(imgPixels);
return contoursAreas;
}
private Boolean isContourQualifiable(Double contourArea, Double imageSize) {
return contourArea > imageSize * 0.01;
}
private void saveDebugImage(Mat imgToWrite, String uid, String operationName, String contentType) {
if(imageProcessingConfigurationProperties.getDebug()) {
imwrite("./" + uid + "__" + operationName + "." + contentType, imgToWrite);
}
Mat mat = new Mat();
System.out.println("matify exec millis: " + (System.currentTimeMillis() - millis));
return imgMat;
}
}

2
src/main/resources/application.properties
View File

@ -1 +1 @@
imgproc.debug = true