src: loading from pic is working :)

src/components/App/App.tsx

@@ -3,12 +3,20 @@ import React from 'react';
 import ImageUploader from 'react-images-upload';
 import MultiCrops from 'react-multi-crops'
 
+// @ts-ignore
 import Logo from '../Logo/Logo';
+// @ts-ignore
 import Board from '../Board/Board';
 
 import './App.scss';
-//import {findTextRegions} from "../../helpers/findTextRegions";
+// @ts-ignore
-//import {recognizeTextOnImageGrid} from "../../helpers/recognizeTextOnImage";
+import {findTextRegions} from "../../helpers/findTextRegions";
+// @ts-ignore
+import {
+  recognizeTextOnImage,
+// @ts-ignore
+  recognizeTextOnImageGrid
+} from "../../helpers/recognizeTextOnImage";
 
 export type Point = {
   x: number;
@@ -39,6 +47,8 @@ interface IAppProps {
 interface IAppState {
   image: HTMLImageElement | null,
   selections: Array<any>;
+  cells: Array<Array<string>>;
+  solutions: Array<Solution>;
 }
 
 export default class App extends React.Component<IAppProps, IAppState> {
@@ -53,6 +63,8 @@ export default class App extends React.Component<IAppProps, IAppState> {
     this.state = {
       image: null,
       selections: [],
+      cells: [],
+      solutions: [],
     }
   }
 
@@ -60,19 +72,104 @@ export default class App extends React.Component<IAppProps, IAppState> {
     const image = document.createElement('img');
     image.src = URL.createObjectURL(pictures[0]);
     image.onload = () => this.setState({...this.state, image});
-      //this.leftMenu.current;
-      //const r = findTextRegions(image);
-      //console.log(r);
-      //URL.revokeObjectURL(url);
-      //if (r !== null) {
-      //  recognizeTextOnImageGrid(r.grid).then(grid => {
-
-      //  });
-      //}
   }
 
   handleChangeCoordinate(_: any, __: any, selections: any) {
-    this.setState({selections});
+    this.setState({...this.state, selections});
+  }
+
+  handleConfirmClick() {
+    if (!this.state.image || !this.state.selections.length)
+      return;
+
+    const mainCanvas = document.createElement('canvas');
+    const mainContext = mainCanvas.getContext('2d');
+    mainCanvas.width = this.state.image.width;
+    mainCanvas.height = this.state.image.height;
+    mainContext?.drawImage(this.state.image, 0, 0);
+
+    const areas = this.state.selections.map(s => s.width * s.height);
+    const gridIndex = areas.indexOf(Math.max(...areas));
+    const gridSelection = this.state.selections.splice(gridIndex, 1)[0];
+    const gridImage = mainContext?.getImageData(
+      gridSelection.x,
+      gridSelection.y,
+      gridSelection.width,
+      gridSelection.height
+    );
+
+    if (!gridImage) {
+      console.log("gridImage is empty");
+      return;
+    }
+
+    const gridTextRegions = findTextRegions(gridImage);
+    if (!gridTextRegions || !gridTextRegions.grid) {
+      console.log("gridRegions is empty");
+      return;
+    }
+
+    let reads = [];
+    reads.push(this.handleGridTextRegionsReadyToRead(gridTextRegions.grid));
+
+    for (let s of this.state.selections) {
+      const tempData = mainContext?.getImageData(s.x, s.y, s.width, s.height);
+      if (!tempData) {
+        console.log("tempData is empty");
+        continue;
+      }
+
+      reads.push(this.handleStateSelectionsReadyToRead(tempData, s))
+    }
+
+    Promise.all(reads).then(() => {
+      // free resources
+      if (this.state.image) {
+        URL.revokeObjectURL(this.state.image.src);
+        mainCanvas.width = 0;
+        mainCanvas.height = 0;
+      }
+
+      this.setState({
+        ...this.state,
+        image: null,
+        selections: [],
+      });
+    });
+  }
+
+  async handleGridTextRegionsReadyToRead(grid: any) {
+    const textGrid = await recognizeTextOnImageGrid(grid);
+    let cells = [];
+
+    for (let row of textGrid) {
+      let line = [];
+
+      for (let letter of row) {
+        line.push(letter.text);
+      }
+
+      cells.push(line);
+    }
+
+    this.setState({...this.state, cells});
+  }
+
+  async handleStateSelectionsReadyToRead(data: ImageData, s: any) {
+    const tempCanvas = document.createElement('canvas');
+    const tempContext = tempCanvas.getContext('2d');
+    tempCanvas.width = s.width;
+    tempCanvas.height = s.height;
+    tempContext?.putImageData(data, 0, 0);
+
+    const text = await recognizeTextOnImage(tempCanvas);
+    let solutions = this.state.solutions.slice();
+
+    for (let key of text.split("\n")) {
+      solutions.push({key, selection: {start: {x: 0, y: 0}, end: {x: 0, y: 0}}});
+    }
+
+    this.setState({...this.state, solutions});
   }
 
   renderAnswers(): Array<JSX.Element> {
@@ -103,6 +200,7 @@ export default class App extends React.Component<IAppProps, IAppState> {
             />
             <button
               className="ConfirmButton"
+              onClick={this.handleConfirmClick.bind(this)}
               style={{visibility: this.state.image ? 'visible' : 'hidden'}}
             >Potwierdź</button>
           </div>

src/helpers/findTextRegions.js

@@ -15,7 +15,7 @@ export function findTextRegions(image, maxWhiteSpace, maxFontLineWidth, minTextW
   canvas.height = image.height;
 
   let ctx = canvas.getContext('2d');
-  ctx.drawImage(image, 0, 0);
+  ctx.putImageData(image, 0, 0);
   let data = ctx.getImageData(0, 0, image.width, image.height);
 
   // convert to black & white picture

src/helpers/recognizeTextOnImage.ts

@@ -9,7 +9,7 @@ async function prepareWorker(isColumn: boolean = true): Promise<Worker> {
     await worker.loadLanguage('pol');
     await worker.initialize('pol');
     await worker.setParameters({
-      // @ts-ignore, i don't know why this PSM.... stuff is mad at me
+      // @ts-ignore, i don't know why this PSM... stuff is mad at me
       tessedit_pageseg_mode: isColumn ? PSM.SINGLE_COLUMN : PSM.SINGLE_CHAR,
       tessedit_char_whitelist: 'ABCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWXYZŹŻ', // no 'Ą'!
       tessjs_create_box: '1',
@@ -20,6 +20,10 @@ async function prepareWorker(isColumn: boolean = true): Promise<Worker> {
 
 async function recognize(worker: Worker, image: ImageLike) {
   // bug fix - letter 'I' (capital i) is often not recognised
+  // @ts-ignore
+  document.image = image;
+  // @ts-ignore
+  document.recognize = worker.recognize;
   return (await worker.recognize(image)).data.text.trim() || "I";
 }
 
@@ -35,7 +39,6 @@ export async function recognizeTextOnImageGrid(grid: any[][]): Promise<any[][]>
   for (let line of grid) {
     for (let g of line) {
       g.text = await recognize(CHAR_WORKER, g.canvas);
-      g.data = g.canvas = null;  // free mem
     }
   }