s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added ignore:true to any triple that shouldn't be loaded.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1095 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-07-10 05:07:24 +00:00
parent fc005d2958
commit 43dadf40da
4 changed files with 96 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
main/src/com/metaweb/gridworks/protograph/CellTopicNode.java
View File

@ -8,18 +8,15 @@ import org.json.JSONException;
import org.json.JSONWriter; import org.json.JSONWriter;
public class CellTopicNode extends CellNode implements NodeWithLinks { public class CellTopicNode extends CellNode implements NodeWithLinks {
final public boolean createForNoReconMatch;
final public FreebaseType type; final public FreebaseType type;
final public List<Link> links = new LinkedList<Link>(); final public List<Link> links = new LinkedList<Link>();
public CellTopicNode( public CellTopicNode(
String columnName, String columnName,
boolean createForNoReconMatch,
FreebaseType type FreebaseType type
) { ) {
super(columnName); super(columnName);
this.createForNoReconMatch = createForNoReconMatch;
this.type = type; this.type = type;
} }
@ -29,8 +26,7 @@ public class CellTopicNode extends CellNode implements NodeWithLinks {
writer.object(); writer.object();
writer.key("nodeType"); writer.value("cell-as-topic"); writer.key("nodeType"); writer.value("cell-as-topic");
writer.key("columnName"); writer.value(columnName); writer.key("columnName"); writer.value(columnName);
writer.key("createForNoReconMatch"); writer.value(createForNoReconMatch); if (type != null) {
if (createForNoReconMatch && type != null) {
writer.key("type"); type.write(writer, options); writer.key("type"); type.write(writer, options);
} }
if (links != null) { if (links != null) {

1
main/src/com/metaweb/gridworks/protograph/Protograph.java
View File

@ -51,7 +51,6 @@ public class Protograph implements OverlayModel {
if (o.has("type")) { if (o.has("type")) {
node = new CellTopicNode( node = new CellTopicNode(
columnName, columnName,
o.getBoolean("createForNoReconMatch"),
reconstructType(o.getJSONObject("type")) reconstructType(o.getJSONObject("type"))
); );
} }

3
main/src/com/metaweb/gridworks/protograph/transpose/Transposer.java
View File

@ -107,8 +107,7 @@ public class Transposer {
Column column = project.columnModel.getColumnByName(node2.columnName); Column column = project.columnModel.getColumnByName(node2.columnName);
Cell cell = row.getCell(column.getCellIndex()); Cell cell = row.getCell(column.getCellIndex());
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
if (node2 instanceof CellTopicNode && if (node2 instanceof CellTopicNode &&
!((CellTopicNode) node2).createForNoReconMatch &&
(cell.recon == null || cell.recon.judgment == Judgment.None)) { (cell.recon == null || cell.recon.judgment == Judgment.None)) {
return; return;
} }

179
main/src/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java
View File

@ -97,7 +97,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
sb2.append(Long.toString(recon.id)); sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : "); sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
sb2.append(JSONObject.quote(column.getName())); sb2.append(JSONObject.quote(column.getName()));
sb2.append(" }"); sb2.append(", \"ignore\" : true }");
writeLine(sb2.toString()); writeLine(sb2.toString());
} }
@ -110,7 +110,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
sb2.append("{ \"s\" : \"rec"); sb2.append("{ \"s\" : \"rec");
sb2.append(Long.toString(recon.id)); sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:recon_data\", \"o\" : { "); sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry)); sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s)); sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
@ -135,7 +135,8 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
String subject, String predicate, Object object, String subject, String predicate, Object object,
Project project, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
int objectRowIndex, int objectCellIndex, Cell objectCell int objectRowIndex, int objectCellIndex, Cell objectCell,
boolean ignore
) { ) {
if (subject != null && object != null) { if (subject != null && object != null) {
String s = object instanceof String ? String s = object instanceof String ?
@ -164,6 +165,9 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
sb.append(" }"); sb.append(" }");
} }
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }"); sb.append(" }");
writeLine(sb.toString()); writeLine(sb.toString());
@ -172,7 +176,8 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
protected void writeLine( protected void writeLine(
String subject, String predicate, Object object, String lang, String subject, String predicate, Object object, String lang,
Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
boolean ignore
) { ) {
if (subject != null && object != null) { if (subject != null && object != null) {
String s = object instanceof String ? String s = object instanceof String ?
@ -192,6 +197,9 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
sb.append(" }"); sb.append(" }");
sb.append(" }"); sb.append(" }");
} }
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }"); sb.append(" }");
writeLine(sb.toString()); writeLine(sb.toString());
@ -205,7 +213,10 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public Object write( public Object write(
String subject, String predicate, Project project, String subject, String predicate, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
return load ? internalWrite(subject, predicate, project, subjectRowIndex, subjectCellIndex, subjectCell) : null;
return internalWrite(
subject, predicate, project,
subjectRowIndex, subjectCellIndex, subjectCell);
} }
abstract public Object internalWrite( abstract public Object internalWrite(
@ -218,15 +229,17 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public List<Integer> rowIndices = new LinkedList<Integer>(); public List<Integer> rowIndices = new LinkedList<Integer>();
public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>(); public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
protected void writeChildren(String subject, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { protected void writeChildren(
String subject, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
for (int i = 0; i < children.size(); i++) { for (int i = 0; i < children.size(); i++) {
WritingTransposedNode child = children.get(i); WritingTransposedNode child = children.get(i);
Link link = links.get(i); Link link = links.get(i);
if (link.load) { String predicate = link.property.id;
String predicate = link.property.id;
child.write(subject, predicate, project,
child.write(subject, predicate, project, subjectRowIndex, subjectCellIndex, subjectCell); subjectRowIndex, subjectCellIndex, subjectCell);
}
} }
} }
} }
@ -261,38 +274,36 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
WritingTransposedNode child = children.get(i); WritingTransposedNode child = children.get(i);
Link link = links.get(i); Link link = links.get(i);
if (link.load) { FreebaseProperty property = link.property;
FreebaseProperty property = link.property;
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null); if (c != null) {
if (c != null) { if (first) {
if (first) { first = false;
first = false; } else {
} else { sb.append(", ");
sb.append(", ");
}
sb.append("\"" + property.id + "\": ");
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
} }
sb.append("\"" + property.id + "\": ");
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
}
if (child instanceof CellTopicTransposedNode) {
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
Recon recon = child2.cell.recon;
if (child instanceof CellTopicTransposedNode) { if (recon != null &&
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child; (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
Recon recon = child2.cell.recon;
if (recon != null && if (firstRecon) {
(recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) { firstRecon = false;
} else {
if (firstRecon) { sbRecon.append(", ");
firstRecon = false;
} else {
sbRecon.append(", ");
}
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
writeRecon(sbRecon, project,
rowIndices.get(i), child2.cellIndex, child2.cell);
} }
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
writeRecon(sbRecon, project,
rowIndices.get(i), child2.cellIndex, child2.cell);
} }
} }
} }
@ -321,50 +332,46 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
String id = null; String id = null;
int objectRowIndex = -1; if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
int objectCellIndex = -1; int objectRowIndex = rowIndex;
Cell objectCell = null; int objectCellIndex = cellIndex;
Cell objectCell = cell;
if (cell.recon != null &&
cell.recon.judgment == Recon.Judgment.Matched &&
cell.recon.match != null) {
objectRowIndex = rowIndex; if (cell.recon.judgment == Recon.Judgment.Matched) {
objectCellIndex = cellIndex; id = cell.recon.match.id;
objectCell = cell;
id = cell.recon.match.id; } else if (cell.recon.judgment == Judgment.New) {
} else if (node.createForNoReconMatch || if (newTopicVars.containsKey(cell.recon.id)) {
(cell.recon != null && cell.recon.judgment == Judgment.New)) { id = newTopicVars.get(cell.recon.id);
if (cell.recon != null && newTopicVars.containsKey(cell.recon.id)) { } else {
id = newTopicVars.get(cell.recon.id); long var = 0;
if (varPool.containsKey(node.columnName)) {
var = varPool.get(node.columnName);
}
varPool.put(node.columnName, var + 1);
id = "$" + node.columnName.replaceAll("\\W+", "_") + "_" + var;
writeLine(id, "type", node.type.id, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
if (cell.recon != null) {
newTopicVars.put(cell.recon.id, id);
}
}
} else { } else {
long var = 0; return null;
if (varPool.containsKey(node.columnName)) {
var = varPool.get(node.columnName);
}
varPool.put(node.columnName, var + 1);
id = "$" + node.columnName.replaceAll("\\W+", "_") + "_" + var;
writeLine(id, "type", node.type.id, project, -1, -1, (Cell) null, -1, -1, (Cell) null);
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null);
if (cell.recon != null) {
newTopicVars.put(cell.recon.id, id);
}
} }
} else {
return null; if (subject != null) {
writeLine(subject, predicate, id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
objectRowIndex, objectCellIndex, objectCell, !load);
}
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
} }
if (subject != null) {
writeLine(subject, predicate, id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
objectRowIndex, objectCellIndex, objectCell);
}
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
return id; return id;
} }
} }
@ -387,11 +394,11 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
if (subject != null) { if (subject != null) {
if ("/type/text".equals(node.lang)) { if ("/type/text".equals(node.lang)) {
writeLine(subject, predicate, cell.value, node.lang, project, writeLine(subject, predicate, cell.value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell); subjectRowIndex, subjectCellIndex, subjectCell, !load);
} else { } else {
writeLine(subject, predicate, cell.value, project, writeLine(subject, predicate, cell.value, project,
subjectRowIndex, subjectCellIndex, subjectCell, subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null); -1, -1, null, !load);
} }
} }
@ -415,7 +422,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, "key", node.namespace.id + "/" + cell.value, project, writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
subjectRowIndex, subjectCellIndex, subjectCell, subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null); -1, -1, null, !load);
return null; return null;
} }
@ -431,7 +438,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, predicate, node.topic.id, project, writeLine(subject, predicate, node.topic.id, project,
subjectRowIndex, subjectCellIndex, subjectCell, subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null); -1, -1, null, !load);
writeChildren(node.topic.id, project, -1, -1, null); writeChildren(node.topic.id, project, -1, -1, null);
@ -449,11 +456,11 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
if ("/type/text".equals(node.lang)) { if ("/type/text".equals(node.lang)) {
writeLine(subject, predicate, node.value, node.lang, project, writeLine(subject, predicate, node.value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell); subjectRowIndex, subjectCellIndex, subjectCell, !load);
} else { } else {
writeLine(subject, predicate, node.value, project, writeLine(subject, predicate, node.value, project,
subjectRowIndex, subjectCellIndex, subjectCell, subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null); -1, -1, null, !load);
} }
return node.value; return node.value;
@ -632,7 +639,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
{ {
StringBuffer sb2 = new StringBuffer(); StringBuffer sb2 = new StringBuffer();
sb2.append("{ \"s\" : \"rec"); sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
sb2.append(Long.toString(cell.recon.id)); sb2.append(Long.toString(cell.recon.id));
sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx"); sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
sb2.append(Long.toString(contextID)); sb2.append(Long.toString(contextID));
@ -688,7 +695,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
if (contextTreeRoot != null && contextRefCount > 0) { if (contextTreeRoot != null && contextRefCount > 0) {
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \"ctx"); sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
sb.append(Long.toString(contextID)); sb.append(Long.toString(contextID));
sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : "); sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
sb.append(Integer.toString(contextRowIndex)); sb.append(Integer.toString(contextRowIndex));