s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added new recon commands:

Browse Source 
- clear recon data for all matching rows
- clear recon data for one cell
- clear recon data for similar cells
- copy recon judgments across columns

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1618 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-10-22 07:25:27 +00:00
parent 91797339e0
commit 345c1c62ac
13 changed files with 846 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java Normal file
View File

@ -0,0 +1,172 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.commands.recon;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.changes.CellChange;
import com.google.refine.model.changes.ReconChange;
import com.google.refine.process.QuickHistoryEntryProcess;
import com.google.refine.util.Pool;
public class ReconClearOneCellCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
int rowIndex = Integer.parseInt(request.getParameter("row"));
int cellIndex = Integer.parseInt(request.getParameter("cell"));
ClearOneCellProcess process = new ClearOneCellProcess(
project,
"Clear one cell's recon data",
rowIndex,
cellIndex
);
HistoryEntry historyEntry = project.processManager.queueProcess(process);
if (historyEntry != null) {
/*
* If the process is done, write back the cell's data so that the
* client side can update its UI right away.
*/
JSONWriter writer = new JSONWriter(response.getWriter());
Pool pool = new Pool();
Properties options = new Properties();
options.put("pool", pool);
writer.object();
writer.key("code"); writer.value("ok");
writer.key("historyEntry"); historyEntry.write(writer, options);
writer.key("cell"); process.newCell.write(writer, options);
writer.key("pool"); pool.write(writer, options);
writer.endObject();
} else {
respond(response, "{ \"code\" : \"pending\" }");
}
} catch (Exception e) {
respondException(response, e);
}
}
protected static class ClearOneCellProcess extends QuickHistoryEntryProcess {
final int rowIndex;
final int cellIndex;
Cell newCell;
ClearOneCellProcess(
Project project,
String briefDescription,
int rowIndex,
int cellIndex
) {
super(project, briefDescription);
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
}
protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception {
Cell cell = _project.rows.get(rowIndex).getCell(cellIndex);
if (cell == null || !ExpressionUtils.isNonBlankData(cell.value)) {
throw new Exception("Cell is blank or error");
}
Column column = _project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
throw new Exception("No such column");
}
Judgment oldJudgment = cell.recon == null ? Judgment.None : cell.recon.judgment;
newCell = new Cell(cell.value, null);
ReconStats stats = column.getReconStats();
if (stats == null) {
stats = ReconStats.create(_project, cellIndex);
} else {
int newChange = 0;
int matchChange = 0;
if (oldJudgment == Judgment.New) {
newChange--;
}
if (oldJudgment == Judgment.Matched) {
matchChange--;
}
stats = new ReconStats(
stats.nonBlanks + 1,
stats.newTopics + newChange,
stats.matchedTopics + matchChange);
}
String description =
"Clear recon data for single cell on row " + (rowIndex + 1) +
", column " + column.getName() +
", containing \"" + cell.value + "\"";
Change change = new ReconChange(
new CellChange(rowIndex, cellIndex, cell, newCell),
column.getName(),
column.getReconConfig(),
stats
);
return new HistoryEntry(
historyEntryID, _project, description, null, change);
}
}
}

60
main/src/com/google/refine/commands/recon/ReconClearSimilarCellsCommand.java Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.commands.recon;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.recon.ReconClearSimilarCellsOperation;
public class ReconClearSimilarCellsCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(
Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception {
String columnName = request.getParameter("columnName");
String similarValue = request.getParameter("similarValue");
return new ReconClearSimilarCellsOperation(
engineConfig,
columnName,
similarValue
);
}
}

58
main/src/com/google/refine/commands/recon/ReconCopyAcrossColumnsCommand.java Normal file
View File

@ -0,0 +1,58 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.commands.recon;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.recon.ReconCopyAcrossColumnsOperation;
public class ReconCopyAcrossColumnsCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project,
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String fromColumnName = request.getParameter("fromColumnName");
String[] toColumnNames = request.getParameterValues("toColumnName[]");
String[] judgments = request.getParameterValues("judgment[]");
boolean applyToJudgedCells = Boolean.parseBoolean(request.getParameter("applyToJudgedCells"));
return new ReconCopyAcrossColumnsOperation(
engineConfig, fromColumnName, toColumnNames, judgments, applyToJudgedCells);
}
}

3
main/src/com/google/refine/commands/recon/ReconDiscardJudgmentsCommand.java
View File

@ -48,7 +48,8 @@ public class ReconDiscardJudgmentsCommand extends EngineDependentCommand {
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String columnName = request.getParameter("columnName");
boolean clearData = Boolean.parseBoolean(request.getParameter("clearData"));
return new ReconDiscardJudgmentsOperation(engineConfig, columnName);
return new ReconDiscardJudgmentsOperation(engineConfig, columnName, clearData);
}
}

142
main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java Normal file
View File

@ -0,0 +1,142 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.operations.recon;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.model.changes.ReconChange;
import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
public class ReconClearSimilarCellsOperation extends EngineDependentMassCellOperation {
final protected String _similarValue;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ReconClearSimilarCellsOperation(
engineConfig,
obj.getString("columnName"),
obj.getString("similarValue")
);
}
public ReconClearSimilarCellsOperation(
JSONObject engineConfig,
String columnName,
String similarValue
) {
super(engineConfig, columnName, false);
this._similarValue = similarValue;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("columnName"); writer.value(_columnName);
writer.key("similarValue"); writer.value(_similarValue);
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Clear recon data for cells containing \"" +
_similarValue + "\" in column " + _columnName;
}
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Clear recon data for " + cellChanges.size() + " cells containing \"" +
_similarValue + "\" in column " + _columnName;
}
protected RowVisitor createRowVisitor(final Project project, final List<CellChange> cellChanges, final long historyEntryID) throws Exception {
Column column = project.columnModel.getColumnByName(_columnName);
final int cellIndex = column != null ? column.getCellIndex() : -1;
return new RowVisitor() {
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = cellIndex < 0 ? null : row.getCell(cellIndex);
if (cell != null && cell.recon != null) {
String value = cell.value instanceof String ?
((String) cell.value) : cell.value.toString();
if (_similarValue.equals(value)) {
Cell newCell = new Cell(cell.value, null);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
return false;
}
};
}
protected Change createChange(Project project, Column column, List<CellChange> cellChanges) {
return new ReconChange(
cellChanges,
_columnName,
column.getReconConfig(),
null
);
}
}

212
main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java Normal file
View File

@ -0,0 +1,212 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.operations.recon;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.model.changes.MassChange;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.JSONUtilities;
public class ReconCopyAcrossColumnsOperation extends EngineDependentOperation {
final protected String _fromColumnName;
final protected String[] _toColumnNames;
final protected String[] _judgments;
final protected boolean _applyToJudgedCells;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ReconCopyAcrossColumnsOperation(
engineConfig,
obj.getString("fromColumnName"),
JSONUtilities.getStringArray(obj, "toColumnNames"),
JSONUtilities.getStringArray(obj, "judgments"),
obj.getBoolean("applyToJudgedCells")
);
}
public ReconCopyAcrossColumnsOperation(
JSONObject engineConfig,
String fromColumnName,
String[] toColumnNames,
String[] judgments,
boolean applyToJudgedCells) {
super(engineConfig);
_fromColumnName = fromColumnName;
_toColumnNames = toColumnNames;
_judgments = judgments;
_applyToJudgedCells = applyToJudgedCells;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("fromColumnName"); writer.value(_fromColumnName);
writer.key("toColumnNames");
writer.array();
for (String s : _toColumnNames) {
writer.value(s);
}
writer.endArray();
writer.key("judgments");
writer.array();
for (String s : _judgments) {
writer.value(s);
}
writer.endArray();
writer.key("applyToJudgedCells"); writer.value(_applyToJudgedCells);
writer.endObject();
}
protected HistoryEntry createHistoryEntry(final Project project, final long historyEntryID) throws Exception {
Engine engine = createEngine(project);
final Column fromColumn = project.columnModel.getColumnByName(_fromColumnName);
final List<Column> toColumns = new ArrayList<Column>(_toColumnNames.length);
for (String c : _toColumnNames) {
Column toColumn = project.columnModel.getColumnByName(c);
if (toColumn != null) {
toColumns.add(toColumn);
}
}
final Set<Recon.Judgment> judgments = new HashSet<Recon.Judgment>(_judgments.length);
for (String j : _judgments) {
judgments.add(Recon.stringToJudgment(j));
}
final List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
if (fromColumn != null && toColumns.size() > 0) {
final Map<Object, Recon> cellValueToRecon = new HashMap<Object, Recon>();
FilteredRows filteredRows = engine.getAllFilteredRows();
try {
filteredRows.accept(project, new RowVisitor() {
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(fromColumn.getCellIndex());
if (cell != null && cell.value != null && cell.recon != null) {
if (judgments.contains(cell.recon.judgment)) {
cellValueToRecon.put(cell.value, cell.recon);
}
}
return false;
}
});
filteredRows.accept(project, new RowVisitor() {
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
for (Column column : toColumns) {
int cellIndex = column.getCellIndex();
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.value != null) {
Recon reconToCopy = cellValueToRecon.get(cell.value);
boolean judged = cell.recon != null && cell.recon.judgment != Judgment.None;
if (reconToCopy != null && (!judged || _applyToJudgedCells)) {
Cell newCell = new Cell(cell.value, reconToCopy);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
}
return false;
}
});
} catch (Exception e) {
e.printStackTrace();
}
}
String description = "Copy " + cellChanges.size() + " recon judgments from column " +
_fromColumnName + " to " + StringUtils.join(_toColumnNames);
return new HistoryEntry(
historyEntryID, project, description, this, new MassChange(cellChanges, false));
}
protected String getBriefDescription(Project project) {
return "Copy recon judgments from column " +
_fromColumnName + " to " + StringUtils.join(_toColumnNames);
}
}

49
main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java
View File

@ -57,18 +57,20 @@ import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOperation {
final protected boolean _clearData;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
String columnName = obj.getString("columnName");
return new ReconDiscardJudgmentsOperation(
engineConfig,
columnName
obj.getString("columnName"),
obj.has("clearData") && obj.getBoolean("clearData")
);
}
public ReconDiscardJudgmentsOperation(JSONObject engineConfig, String columnName) {
public ReconDiscardJudgmentsOperation(JSONObject engineConfig, String columnName, boolean clearData) {
super(engineConfig, columnName, false);
_clearData = clearData;
}
public void write(JSONWriter writer, Properties options)
@ -79,18 +81,23 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("columnName"); writer.value(_columnName);
writer.key("clearData"); writer.value(_clearData);
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Discard recon judgments for cells in column " + _columnName;
return _clearData ?
"Discard recon judgments and clear recon data for cells in column " + _columnName :
"Discard recon judgments for cells in column " + _columnName;
}
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Discard recon judgments for " + cellChanges.size() +
" cells in column " + column.getName();
return (_clearData ?
"Discard recon judgments and clear recon data" :
"Discard recon judgments") +
" for " + cellChanges.size() + " cells in column " + column.getName();
}
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges, long historyEntryID) throws Exception {
@ -122,19 +129,21 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null) {
Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
newRecon.match = null;
newRecon.matchRank = -1;
newRecon.judgment = Judgment.None;
newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1;
dupReconMap.put(cell.recon.id, newRecon);
Recon newRecon = null;
if (!_clearData) {
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
newRecon.match = null;
newRecon.matchRank = -1;
newRecon.judgment = Judgment.None;
newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1;
dupReconMap.put(cell.recon.id, newRecon);
}
}
Cell newCell = new Cell(cell.value, newRecon);

6
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -102,6 +102,10 @@ function registerCommands() {
RS.registerCommand(module, "recon-match-specific-topic-to-cells", new Packages.com.google.refine.commands.recon.ReconMatchSpecificTopicCommand());
RS.registerCommand(module, "recon-judge-one-cell", new Packages.com.google.refine.commands.recon.ReconJudgeOneCellCommand());
RS.registerCommand(module, "recon-judge-similar-cells", new Packages.com.google.refine.commands.recon.ReconJudgeSimilarCellsCommand());
RS.registerCommand(module, "recon-clear-one-cell", new Packages.com.google.refine.commands.recon.ReconClearOneCellCommand());
RS.registerCommand(module, "recon-clear-similar-cells", new Packages.com.google.refine.commands.recon.ReconClearSimilarCellsCommand());
RS.registerCommand(module, "recon-copy-across-columns", new Packages.com.google.refine.commands.recon.ReconCopyAcrossColumnsCommand());
RS.registerCommand(module, "guess-types-of-column", new Packages.com.google.refine.commands.recon.GuessTypesOfColumnCommand());
RS.registerCommand(module, "annotate-one-row", new Packages.com.google.refine.commands.row.AnnotateOneRowCommand());
@ -152,6 +156,8 @@ function registerOperations() {
OR.registerOperation(module, "recon-discard-judgments", Packages.com.google.refine.operations.recon.ReconDiscardJudgmentsOperation);
OR.registerOperation(module, "recon-match-specific-topic-to-cells", Packages.com.google.refine.operations.recon.ReconMatchSpecificTopicOperation);
OR.registerOperation(module, "recon-judge-similar-cells", Packages.com.google.refine.operations.recon.ReconJudgeSimilarCellsOperation);
OR.registerOperation(module, "recon-clear-similar-cells", Packages.com.google.refine.operations.recon.ReconClearSimilarCellsOperation);
OR.registerOperation(module, "recon-copy-across-columns", Packages.com.google.refine.operations.recon.ReconCopyAcrossColumnsOperation);
}
/*

3
main/webapp/modules/core/scripts/views/data-table/cell-recon-search-for-match.html
View File

@ -19,7 +19,8 @@
</div>
<div class="dialog-footer" bind="dialogFooter">
<button class="button button-primary" bind="okButton">Match</button>
<button class="button" bind="newButton">New</button>
<button class="button" bind="newButton">New Topic</button>
<button class="button" bind="clearButton">Clear</button>
<button class="button" bind="cancelButton">Cancel</button>
</div>
</div>

49
main/webapp/modules/core/scripts/views/data-table/cell-ui.js
View File

@ -208,17 +208,27 @@ DataTableCellUI.prototype._render = function() {
} else if (ReconciliationManager.isFreebaseIdOrMid(r.identifierSpace)) {
addSuggest = true;
}
var extraChoices = $('<div>').addClass("data-table-recon-extra").appendTo(divContent);
if (addSuggest) {
$('<a href="javascript:{}"></a>')
.addClass("data-table-recon-search")
.click(function(evt) {
self._searchForMatch(suggestOptions);
return false;
})
.text("Search for match")
.appendTo($('<div>').appendTo(divContent));
.appendTo(extraChoices);
$('<span>').html(" &bull; ").appendTo(extraChoices);
}
$('<a href="javascript:{}"></a>')
.click(function(evt) {
self._doClearOneCell();
return false;
})
.text("Clear")
.appendTo(extraChoices);
}
}
}
@ -234,10 +244,34 @@ DataTableCellUI.prototype._doMatchNewTopicToOneCell = function() {
this._doJudgment("new");
};
DataTableCellUI.prototype._doClearOneCell = function() {
this._postProcessOneCell(
"recon-clear-one-cell",
{},
{
row: this._rowIndex,
cell: this._cellIndex
},
true
);
};
DataTableCellUI.prototype._doMatchNewTopicToSimilarCells = function() {
this._doJudgmentForSimilarCells("new", {}, { shareNewTopics: true }, true);
};
DataTableCellUI.prototype._doClearSimilarCells = function() {
this._postProcessSeveralCells(
"recon-clear-similar-cells",
{},
{
columnName: Refine.cellIndexToColumn(this._cellIndex).name,
similarValue: this._cell.v
},
true
);
};
DataTableCellUI.prototype._doMatchTopicToOneCell = function(candidate) {
this._doJudgment("matched", {}, {
id : candidate.id,
@ -333,9 +367,18 @@ DataTableCellUI.prototype._searchForMatch = function(suggestOptions) {
}
dismiss();
};
var commitClear = function() {
if (elmts.radioSimilar[0].checked) {
self._doClearSimilarCells();
} else {
self._doClearOneCell();
}
dismiss();
};
elmts.okButton.click(commit);
elmts.newButton.click(commitNew);
elmts.clearButton.click(commitClear);
elmts.cancelButton.click(dismiss);
var suggestOptions2 = $.extend({ align: "left" }, suggestOptions || { all_types: true });

34
main/webapp/modules/core/scripts/views/data-table/copy-recon-across-columns-dialog.html Normal file
View File

@ -0,0 +1,34 @@
<div class="dialog-frame" style="width: 600px;">
<div class="dialog-border">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div class="grid-layout layout-normal layout-full grid-layout-for-ui"><table>
<tr>
<td>Copy to Columns</td>
<td>Copying Options</td>
</tr>
<tr>
<td><select bind="toColumnSelect" multiple size="20" style="width: 100%;"></select></td>
<td><div class="grid-layout layout-normal"><table>
<tr>
<td><input type="checkbox" bind="applyToJudgedCellsCheckbox" checked /></td><td>Apply to judged cells</td>
</tr>
<tr>
<td colspan="2">What to copy:</td>
</tr>
<tr>
<td><input type="checkbox" bind="newCheckbox" checked /></td><td>new recon judgments</td>
</tr>
<tr>
<td><input type="checkbox" bind="matchCheckbox" checked /></td><td>match recon judgments</td>
</tr>
</table></div></td>
</tr>
</table></div>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button class="button" bind="okButton">Copy</button>
<button class="button" bind="cancelButton">Cancel</button>
</div>
</div>
</div>

77
main/webapp/modules/core/scripts/views/data-table/menu-reconcile.js
View File

@ -39,11 +39,20 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
var doReconDiscardJudgments = function() {
Refine.postCoreProcess(
"recon-discard-judgments",
{ columnName: column.name },
{ columnName: column.name, clearData: false },
null,
{ cellsChanged: true, columnStatsChanged: true }
);
};
var doClearReconData = function() {
Refine.postCoreProcess(
"recon-discard-judgments",
{ columnName: column.name, clearData: true },
null,
{ cellsChanged: true, columnStatsChanged: true }
);
}
var doReconMatchBestCandidates = function() {
Refine.postCoreProcess(
@ -115,6 +124,59 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
input.focus().data("suggest").textchange();
};
var doCopyAcrossColumns = function() {
var frame = $(DOM.loadHTML("core", "scripts/views/data-table/copy-recon-across-columns-dialog.html"));
var elmts = DOM.bind(frame);
elmts.dialogHeader.text("Copy recon judgments from column " + column.name);
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column2 = columns[i];
if (column !== column2) {
$('<option>').attr("value", column2.name).text(column2.name).appendTo(elmts.toColumnSelect);
}
}
var level = DialogSystem.showDialog(frame);
var dismiss = function() { DialogSystem.dismissUntil(level - 1); };
elmts.cancelButton.click(dismiss);
elmts.okButton.click(function() {
var config = {
fromColumnName: column.name,
toColumnName: [],
judgment: [],
applyToJudgedCells: elmts.applyToJudgedCellsCheckbox[0].checked
};
if (elmts.newCheckbox[0].checked) {
config.judgment.push("new");
}
if (elmts.matchCheckbox[0].checked) {
config.judgment.push("matched");
}
elmts.toColumnSelect.find("option").each(function() {
if (this.selected) {
config.toColumnName.push(this.value);
}
});
if (config.toColumnName.length == 0) {
alert("Please select some other column to copy to.");
} else if (config.judgment.length == 0) {
alert("Please select at least one kind of judgment to copy.");
} else {
Refine.postCoreProcess(
"recon-copy-across-columns",
null,
config,
{ rowsChanged: true }
);
dismiss();
}
});
};
MenuSystem.appendTo(menu, [ "core/reconcile" ], [
{
id: "core/reconcile",
@ -339,8 +401,21 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
label: "Discard Reconciliation Judgments",
tooltip: "Discard reconciliaton judgments in this column for all current filtered rows",
click: doReconDiscardJudgments
},
{
id: "core/clear-recon-data",
label: "Clear Recon Data",
tooltip: "Clear recon data in this column for all current filtered rows",
click: doClearReconData
}
]
},
{},
{
id: "core/copy-across-columns",
label: "Copy Recon Data ...",
tooltip: "Copy this column's recon data to other columns",
click: doCopyAcrossColumns
}
]);
});

11
main/webapp/modules/core/styles/views/data-table-view.less
View File

@ -200,13 +200,16 @@ a.data-table-recon-topic {
margin: 0 0.5em;
}
a.data-table-recon-action, a.data-table-recon-search {
display: block;
margin: 3px 0 0;
font-size: 0.8em;
a.data-table-recon-action, .data-table-recon-extra > a {
text-decoration: none;
color: @link_secondary;
}
a.data-table-recon-action, .data-table-recon-extra {
font-size: 0.8em;
display: block;
margin: 3px 0 0;
}
a.data-table-recon-match, a.data-table-recon-match-similar {
position: absolute;