From 46a1f7e2bb78ceded006ad7e7b14527c57ae5179 Mon Sep 17 00:00:00 2001 From: Artur Tamborski Date: Sat, 23 Jan 2021 22:35:12 +0100 Subject: [PATCH] src: loading from pic is working :) --- src/components/App/App.tsx | 122 +++++++++++++++++++++++++--- src/helpers/findTextRegions.js | 2 +- src/helpers/recognizeTextOnImage.ts | 11 ++- 3 files changed, 118 insertions(+), 17 deletions(-) diff --git a/src/components/App/App.tsx b/src/components/App/App.tsx index f285b1f..93871e3 100644 --- a/src/components/App/App.tsx +++ b/src/components/App/App.tsx @@ -3,12 +3,20 @@ import React from 'react'; import ImageUploader from 'react-images-upload'; import MultiCrops from 'react-multi-crops' +// @ts-ignore import Logo from '../Logo/Logo'; +// @ts-ignore import Board from '../Board/Board'; import './App.scss'; -//import {findTextRegions} from "../../helpers/findTextRegions"; -//import {recognizeTextOnImageGrid} from "../../helpers/recognizeTextOnImage"; +// @ts-ignore +import {findTextRegions} from "../../helpers/findTextRegions"; +// @ts-ignore +import { + recognizeTextOnImage, +// @ts-ignore + recognizeTextOnImageGrid +} from "../../helpers/recognizeTextOnImage"; export type Point = { x: number; @@ -39,6 +47,8 @@ interface IAppProps { interface IAppState { image: HTMLImageElement | null, selections: Array; + cells: Array>; + solutions: Array; } export default class App extends React.Component { @@ -53,6 +63,8 @@ export default class App extends React.Component { this.state = { image: null, selections: [], + cells: [], + solutions: [], } } @@ -60,19 +72,104 @@ export default class App extends React.Component { const image = document.createElement('img'); image.src = URL.createObjectURL(pictures[0]); image.onload = () => this.setState({...this.state, image}); - //this.leftMenu.current; - //const r = findTextRegions(image); - //console.log(r); - //URL.revokeObjectURL(url); - //if (r !== null) { - // recognizeTextOnImageGrid(r.grid).then(grid => { - - // }); - //} } handleChangeCoordinate(_: any, __: any, selections: any) { - this.setState({selections}); + this.setState({...this.state, selections}); + } + + handleConfirmClick() { + if (!this.state.image || !this.state.selections.length) + return; + + const mainCanvas = document.createElement('canvas'); + const mainContext = mainCanvas.getContext('2d'); + mainCanvas.width = this.state.image.width; + mainCanvas.height = this.state.image.height; + mainContext?.drawImage(this.state.image, 0, 0); + + const areas = this.state.selections.map(s => s.width * s.height); + const gridIndex = areas.indexOf(Math.max(...areas)); + const gridSelection = this.state.selections.splice(gridIndex, 1)[0]; + const gridImage = mainContext?.getImageData( + gridSelection.x, + gridSelection.y, + gridSelection.width, + gridSelection.height + ); + + if (!gridImage) { + console.log("gridImage is empty"); + return; + } + + const gridTextRegions = findTextRegions(gridImage); + if (!gridTextRegions || !gridTextRegions.grid) { + console.log("gridRegions is empty"); + return; + } + + let reads = []; + reads.push(this.handleGridTextRegionsReadyToRead(gridTextRegions.grid)); + + for (let s of this.state.selections) { + const tempData = mainContext?.getImageData(s.x, s.y, s.width, s.height); + if (!tempData) { + console.log("tempData is empty"); + continue; + } + + reads.push(this.handleStateSelectionsReadyToRead(tempData, s)) + } + + Promise.all(reads).then(() => { + // free resources + if (this.state.image) { + URL.revokeObjectURL(this.state.image.src); + mainCanvas.width = 0; + mainCanvas.height = 0; + } + + this.setState({ + ...this.state, + image: null, + selections: [], + }); + }); + } + + async handleGridTextRegionsReadyToRead(grid: any) { + const textGrid = await recognizeTextOnImageGrid(grid); + let cells = []; + + for (let row of textGrid) { + let line = []; + + for (let letter of row) { + line.push(letter.text); + } + + cells.push(line); + } + + this.setState({...this.state, cells}); + } + + async handleStateSelectionsReadyToRead(data: ImageData, s: any) { + const tempCanvas = document.createElement('canvas'); + const tempContext = tempCanvas.getContext('2d'); + tempCanvas.width = s.width; + tempCanvas.height = s.height; + tempContext?.putImageData(data, 0, 0); + + const text = await recognizeTextOnImage(tempCanvas); + let solutions = this.state.solutions.slice(); + + for (let key of text.split("\n")) { + solutions.push({key, selection: {start: {x: 0, y: 0}, end: {x: 0, y: 0}}}); + } + + this.setState({...this.state, solutions}); } renderAnswers(): Array { @@ -103,6 +200,7 @@ export default class App extends React.Component { /> diff --git a/src/helpers/findTextRegions.js b/src/helpers/findTextRegions.js index e8fcd75..f9769ca 100644 --- a/src/helpers/findTextRegions.js +++ b/src/helpers/findTextRegions.js @@ -15,7 +15,7 @@ export function findTextRegions(image, maxWhiteSpace, maxFontLineWidth, minTextW canvas.height = image.height; let ctx = canvas.getContext('2d'); - ctx.drawImage(image, 0, 0); + ctx.putImageData(image, 0, 0); let data = ctx.getImageData(0, 0, image.width, image.height); // convert to black & white picture diff --git a/src/helpers/recognizeTextOnImage.ts b/src/helpers/recognizeTextOnImage.ts index a8d5d6c..9bb7267 100644 --- a/src/helpers/recognizeTextOnImage.ts +++ b/src/helpers/recognizeTextOnImage.ts @@ -9,7 +9,7 @@ async function prepareWorker(isColumn: boolean = true): Promise { await worker.loadLanguage('pol'); await worker.initialize('pol'); await worker.setParameters({ - // @ts-ignore, i don't know why this PSM.... stuff is mad at me + // @ts-ignore, i don't know why this PSM... stuff is mad at me tessedit_pageseg_mode: isColumn ? PSM.SINGLE_COLUMN : PSM.SINGLE_CHAR, tessedit_char_whitelist: 'ABCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWXYZŹŻ', // no 'Ą'! tessjs_create_box: '1', @@ -20,12 +20,16 @@ async function prepareWorker(isColumn: boolean = true): Promise { async function recognize(worker: Worker, image: ImageLike) { // bug fix - letter 'I' (capital i) is often not recognised + // @ts-ignore + document.image = image; + // @ts-ignore + document.recognize = worker.recognize; return (await worker.recognize(image)).data.text.trim() || "I"; } export async function recognizeTextOnImage(image: ImageLike): Promise { - TEXT_WORKER = TEXT_WORKER ?? await prepareWorker(true); - return await recognize(TEXT_WORKER, image); + TEXT_WORKER = TEXT_WORKER ?? await prepareWorker(true); + return await recognize(TEXT_WORKER, image); } export async function recognizeTextOnImageGrid(grid: any[][]): Promise { @@ -35,7 +39,6 @@ export async function recognizeTextOnImageGrid(grid: any[][]): Promise for (let line of grid) { for (let g of line) { g.text = await recognize(CHAR_WORKER, g.canvas); - g.data = g.canvas = null; // free mem } }