Tripleloader protograph transposer now generates more context information for QA.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1073 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/metaweb/gridworks/protograph/Link.java

@@ -8,12 +8,14 @@ import org.json.JSONWriter;
 import com.metaweb.gridworks.Jsonizable;
 
 public class Link implements Jsonizable {
-    final public FreebaseProperty     property;
+    final public FreebaseProperty    property;
     final public Node                target;
+    final public boolean             load;
     
-    public Link(FreebaseProperty property, Node target) {
+    public Link(FreebaseProperty property, Node target, boolean load) {
         this.property = property;
         this.target = target;
+        this.load = load;
     }
     
     public FreebaseProperty getProperty() {

main/src/com/metaweb/gridworks/protograph/Protograph.java

@@ -93,7 +93,9 @@ public class Protograph implements Jsonizable {
                 node2.addLink(new Link(
                     reconstructProperty(oLink.getJSONObject("property")),
                     oLink.has("target") && !oLink.isNull("target") ? 
-                        reconstructNode(oLink.getJSONObject("target")) : null
+                        reconstructNode(oLink.getJSONObject("target")) : null,
+                    oLink.has("load") && !oLink.isNull("load") ?
+                        oLink.getBoolean("load") : true
                 ));
             }
         }

main/src/com/metaweb/gridworks/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java

@@ -19,6 +19,7 @@ import com.metaweb.gridworks.protograph.CellTopicNode;
 import com.metaweb.gridworks.protograph.CellValueNode;
 import com.metaweb.gridworks.protograph.FreebaseProperty;
 import com.metaweb.gridworks.protograph.FreebaseTopicNode;
+import com.metaweb.gridworks.protograph.Link;
 import com.metaweb.gridworks.protograph.ValueNode;
 import com.metaweb.gridworks.util.JSONUtilities;
 
@@ -35,7 +36,7 @@ public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory
     private static final String LANG = "lang";
     
     public MqlwriteLikeTransposedNodeFactory(Writer writer) {
-    	this.writer = writer;
+        this.writer = writer;
     }
     
     protected JSONArray getJSON() {
@@ -44,19 +45,19 @@ public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory
     
     @Override
     public void flush() throws IOException {
-    	try {
+        try {
-        	JSONWriter jsonWriter = new JSONWriter(writer);
+            JSONWriter jsonWriter = new JSONWriter(writer);
-        	
+            
-        	jsonWriter.array();
+            jsonWriter.array();
-        	for (JSONObject obj : rootObjects) {
+            for (JSONObject obj : rootObjects) {
-        		jsonWriter.value(obj);
+                jsonWriter.value(obj);
-        	}
+            }
-        	jsonWriter.endArray();
+            jsonWriter.endArray();
-        	
+            
-		} catch (JSONException e) {
+        } catch (JSONException e) {
-			e.printStackTrace();
+            e.printStackTrace();
-		}
+        }
-    	writer.flush();
+        writer.flush();
     }
     
     abstract protected class JsonTransposedNode implements TransposedNode {
@@ -152,8 +153,8 @@ public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory
             if (obj == null) {
                 obj = new JSONObject();
                 try {
-                	JSONUtilities.putField(obj, VALUE, cell.value);
+                    JSONUtilities.putField(obj, VALUE, cell.value);
-                	
+                    
                     obj.put(TYPE, node.valueType);
                     if ("/type/text".equals(node.valueType)) {
                         obj.put(LANG, node.lang);
@@ -246,19 +247,19 @@ public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory
     }    
     public TransposedNode transposeAnonymousNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             AnonymousNode node, int rowIndex) {
         
         return new AnonymousTransposedNode(
             parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null, 
-            property, 
+            link != null ? link.property : null,
             node
         );
     }
 
     public TransposedNode transposeCellNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             CellNode node, 
             int rowIndex, Cell cell) {
         
@@ -272,31 +273,31 @@ public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory
         }
         
         if (tnode != null) {
-            processTransposedNode(tnode, parentNode, property);
+            processTransposedNode(tnode, parentNode, link != null ? link.property : null);
         }
         return tnode;
     }
 
     public TransposedNode transposeTopicNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             FreebaseTopicNode node, int rowIndex) {
         
         JsonTransposedNode tnode = new TopicTransposedNode(node);
         
-        processTransposedNode(tnode, parentNode, property);
+        processTransposedNode(tnode, parentNode, link != null ? link.property : null);
         
         return tnode;
     }
 
     public TransposedNode transposeValueNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             ValueNode node, int rowIndex) {
         
         JsonTransposedNode tnode = new ValueTransposedNode(node);
         
-        processTransposedNode(tnode, parentNode, property);
+        processTransposedNode(tnode, parentNode, link != null ? link.property : null);
         
         return tnode;
     }

main/src/com/metaweb/gridworks/protograph/transpose/TransposedNodeFactory.java

@@ -5,20 +5,20 @@ import java.io.IOException;
 import com.metaweb.gridworks.model.Cell;
 import com.metaweb.gridworks.protograph.AnonymousNode;
 import com.metaweb.gridworks.protograph.CellNode;
-import com.metaweb.gridworks.protograph.FreebaseProperty;
 import com.metaweb.gridworks.protograph.FreebaseTopicNode;
+import com.metaweb.gridworks.protograph.Link;
 import com.metaweb.gridworks.protograph.ValueNode;
 
 public interface TransposedNodeFactory {
     public TransposedNode transposeAnonymousNode(
         TransposedNode parentNode, 
-        FreebaseProperty property, 
+        Link link, 
         AnonymousNode node, int rowIndex
     );
     
     public TransposedNode transposeCellNode(
         TransposedNode parentNode, 
-        FreebaseProperty property, 
+        Link link, 
         CellNode node, 
         int rowIndex,
         Cell cell
@@ -26,14 +26,14 @@ public interface TransposedNodeFactory {
     
     public TransposedNode transposeValueNode(
         TransposedNode parentNode, 
-        FreebaseProperty property, 
+        Link link, 
         ValueNode node, 
         int rowIndex
     );
     
     public TransposedNode transposeTopicNode(
         TransposedNode parentNode, 
-        FreebaseProperty property, 
+        Link link, 
         FreebaseTopicNode node, 
         int rowIndex
     );

main/src/com/metaweb/gridworks/protograph/transpose/Transposer.java

@@ -14,7 +14,6 @@ import com.metaweb.gridworks.model.Recon.Judgment;
 import com.metaweb.gridworks.protograph.AnonymousNode;
 import com.metaweb.gridworks.protograph.CellNode;
 import com.metaweb.gridworks.protograph.CellTopicNode;
-import com.metaweb.gridworks.protograph.FreebaseProperty;
 import com.metaweb.gridworks.protograph.FreebaseTopicNode;
 import com.metaweb.gridworks.protograph.Link;
 import com.metaweb.gridworks.protograph.Node;
@@ -101,7 +100,7 @@ public class Transposer {
         TransposedNode tnode = null;
         
         TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode;
-        FreebaseProperty property = context.parent == null ? null : context.link.property;
+        Link link = context.parent == null ? null : context.link;
         
         if (node instanceof CellNode) {
             CellNode node2 = (CellNode) node;
@@ -121,7 +120,7 @@ public class Transposer {
                 
                 tnode = nodeFactory.transposeCellNode(
                     parentNode,
-                    property,
+                    link,
                     node2, 
                     rowIndex,
                     cell
@@ -131,21 +130,21 @@ public class Transposer {
             if (node instanceof AnonymousNode) {
                 tnode = nodeFactory.transposeAnonymousNode(
                     parentNode,
-                    property,
+                    link,
                     (AnonymousNode) node,
                     rowIndex
                 );
             } else if (node instanceof FreebaseTopicNode) {
                 tnode = nodeFactory.transposeTopicNode(
                     parentNode,
-                    property,
+                    link,
                     (FreebaseTopicNode) node,
                     rowIndex
                 );
             } else if (node instanceof ValueNode) {
                 tnode = nodeFactory.transposeValueNode(
                     parentNode,
-                    property,
+                    link,
                     (ValueNode) node,
                     rowIndex
                 );
@@ -180,12 +179,12 @@ public class Transposer {
     }
     
     static class Context {
-        TransposedNode     transposedNode;
+        TransposedNode    transposedNode;
         List<Context>     subContexts;
-        Context         parent;
+        Context           parent;
-        Link            link;
+        Link              link;
-        int                count;
+        int               count;
-        int                limit;
+        int               limit;
         
         Context(Node node, Context parent, Link link, int limit) {
             this.parent = parent;

main/src/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java

@@ -9,6 +9,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 
 import com.metaweb.gridworks.model.Cell;
@@ -22,7 +24,9 @@ import com.metaweb.gridworks.protograph.CellNode;
 import com.metaweb.gridworks.protograph.CellTopicNode;
 import com.metaweb.gridworks.protograph.CellValueNode;
 import com.metaweb.gridworks.protograph.FreebaseProperty;
+import com.metaweb.gridworks.protograph.FreebaseTopic;
 import com.metaweb.gridworks.protograph.FreebaseTopicNode;
+import com.metaweb.gridworks.protograph.Link;
 import com.metaweb.gridworks.protograph.ValueNode;
 
 public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
@@ -35,6 +39,11 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
     protected Map<Long, String> newTopicVars = new HashMap<Long, String>();
     protected Set<Long> serializedRecons = new HashSet<Long>();
     
+    protected long contextID = 0;
+    protected int contextRowIndex;
+    protected int contextRefCount = 0;
+    protected JSONObject contextTreeRoot;
+    
     public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
         this.project = project;
         this.writer = writer;
@@ -45,6 +54,8 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         if (lastRootNode != null) {
             lastRootNode.write(null, null, project, -1, -1, null);
             lastRootNode = null;
+            
+            writeContextTreeNode();
         }
     }
     
@@ -61,73 +72,63 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
     }
     
-    protected void writeRecon(StringBuffer sb, Project project, int rowIndex, int cellIndex, Cell cell) {
+    protected void writeRecon(
+        StringBuffer sb, 
+        Project project, 
+        int rowIndex, 
+        int cellIndex, 
+        Cell cell
+    ) {
         Recon recon = cell.recon;
-        Column column = project.columnModel.getColumnByCellIndex(cellIndex);
         
-        sb.append("{ ");
+        sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
-        sb.append("\"id\" : "); sb.append(Long.toString(recon.id));
+        contextRefCount++;
         
         if (!serializedRecons.contains(recon.id)) {
             serializedRecons.add(recon.id);
             
-            String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
+            Column column = project.columnModel.getColumnByCellIndex(cellIndex);
-                    
-            sb.append(", \"history_entry\" : "); sb.append(Long.toString(recon.judgmentHistoryEntry));
-            sb.append(", \"text\" : "); sb.append(JSONObject.quote(s));
-            sb.append(", \"column\" : "); sb.append(JSONObject.quote(column.getName()));
-            sb.append(", \"service\" : "); sb.append(JSONObject.quote(recon.service));
-            sb.append(", \"action\" : "); sb.append(JSONObject.quote(recon.judgmentAction));
-            sb.append(", \"batch\" : "); sb.append(Integer.toString(recon.judgmentBatchSize));
-            sb.append(", \"matchRank\" : "); sb.append(Integer.toString(recon.matchRank));
-        }
-        /*
-        sb.append(", \"row\" : [");
-        {
-            boolean first = true;
-            Row row = project.rows.get(rowIndex);
-            RowDependency rowDependency = project.recordModel.getRowDependency(rowIndex);
-            List<Integer> contextRows = rowDependency.contextRows;
             
-            int maxColumns = project.columnModel.columns.size();
+            // qa:sample_group
-            for (int c = 0; c < maxColumns; c++) {
+            {
-                Column column2 = project.columnModel.columns.get(c);
+                StringBuffer sb2 = new StringBuffer();
-                int cellIndex2 = column2.getCellIndex();
                 
-                if (cellIndex2 != cellIndex) {
+                sb2.append("{ \"s\" : \"rec"); 
-                    Object value = row.getCellValue(cellIndex2);
+                sb2.append(Long.toString(recon.id)); 
-                    if (!ExpressionUtils.isNonBlankData(value) && contextRows != null) {
+                sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
-                        for (int i = contextRows.size() - 1; i >= 0; i--) {
+                sb2.append(JSONObject.quote(column.getName()));
-                            int rowIndex2 = contextRows.get(i);
+                sb2.append(" }");
-                            Row row2 = project.rows.get(rowIndex2);
+                
-                            
+                writeLine(sb2.toString());
-                            value = row2.getCellValue(cellIndex2);
+            }
-                            if (ExpressionUtils.isNonBlankData(value)) {
+            
-                                break;
+            // qa:recon_data
-                            }
+            {
-                        }
+                StringBuffer sb2 = new StringBuffer();
-                    }
+                
+                String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
                     
-                    if (ExpressionUtils.isNonBlankData(value)) {
+                sb2.append("{ \"s\" : \"rec"); 
-                        if (first) {
+                sb2.append(Long.toString(recon.id)); 
-                            first = false;
+                sb2.append("\", \"p\" : \"qa:recon_data\", \"o\" : { ");
-                        } else {
+                
-                            sb.append(",");
+                sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
-                        }
+                sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
-                        
+                sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
-                        String s2 = value instanceof String ? (String) value : value.toString();
+                sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
-                        
+                sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
-                        sb.append("{\"c\":"); sb.append(JSONObject.quote(column2.getName()));
+                sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
-                        sb.append(",\"v\":"); sb.append(JSONObject.quote(s2));
+                
-                        sb.append("}");
+                if (recon.judgment == Judgment.Matched) {
-                    }
+                    sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
+                    sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
                 }
+                
+                sb2.append(" } }");
+                
+                writeLine(sb2.toString());
             }
         }
-        sb.append("]");
-        */
-        
-        sb.append(" }");
     }
     
     protected void writeLine(
@@ -147,17 +148,19 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
             if (subjectCell != null || objectCell != null) {
                 sb.append(", \"meta\" : { ");
                 
+                sb.append("\"recon\" : { ");
                 if (subjectCell != null) {
-                    sb.append("\"srecon\" : ");
+                    sb.append("\"s\" : ");
                     writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
                 }
                 if (objectCell != null) {
                     if (subjectCell != null) {
                         sb.append(", ");
                     }
-                    sb.append("\"orecon\" : ");
+                    sb.append("\"o\" : ");
                     writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
                 }
+                sb.append(" }");
                 
                 sb.append(" }");
             }
@@ -183,9 +186,11 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
                     
             if (subjectCell != null) {
                 sb.append(", \"meta\" : { ");
-                sb.append("\"srecon\" : ");
+                sb.append("\"recon\" : { ");
+                sb.append("\"s\" : ");
                 writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
                 sb.append(" }");
+                sb.append(" }");
             }
             sb.append(" }");
                     
@@ -193,20 +198,35 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
     }
     
-    protected interface WritingTransposedNode extends TransposedNode {
+    abstract protected class WritingTransposedNode implements TransposedNode {
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
+        JSONObject jsonContextNode;
+        boolean load;
+        
+        public Object write(
+                String subject, String predicate, Project project,
+                int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+            return load ? internalWrite(subject, predicate, project, subjectRowIndex, subjectCellIndex, subjectCell) : null;
+        }
+        
+        abstract public Object internalWrite(
+            String subject, String predicate, Project project,
+            int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
     }
     
-    abstract protected class TransposedNodeWithChildren implements WritingTransposedNode {
+    abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
-        public List<FreebaseProperty> properties = new LinkedList<FreebaseProperty>();
+        public List<Link> links = new LinkedList<Link>();
+        public List<Integer> rowIndices = new LinkedList<Integer>();
         public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
         
         protected void writeChildren(String subject, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             for (int i = 0; i < children.size(); i++) {
                 WritingTransposedNode child = children.get(i);
-                String predicate = properties.get(i).id;
+                Link link = links.get(i);
-                
+                if (link.load) {
-                child.write(subject, predicate, project, subjectRowIndex, subjectCellIndex, subjectCell);
+                    String predicate = link.property.id;
+                    
+                    child.write(subject, predicate, project, subjectRowIndex, subjectCellIndex, subjectCell);
+                }
             }
         }
     }
@@ -215,30 +235,72 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         
         //protected AnonymousTransposedNode(AnonymousNode node) { }
         
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             if (children.size() == 0 || subject == null) {
                 return null;
             }
             
             StringBuffer sb = new StringBuffer();
-            sb.append("{ ");
+            sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
+            sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
+            sb.append(", \"o\" : { ");
+            
+            StringBuffer sbRecon = new StringBuffer();
             
             boolean first = true;
+            boolean firstRecon = true;
+            
+            if (subjectCell.recon != null) {
+                sbRecon.append("\"s\" : ");
+                writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
+                
+                firstRecon = false;
+            }
+            
             for (int i = 0; i < children.size(); i++) {
-                Object c = children.get(i).write(null, null, project, subjectRowIndex, subjectCellIndex, null);
+                WritingTransposedNode child = children.get(i);
-                if (c != null) {
+                Link link = links.get(i);
-                    if (first) {
+                
-                        first = false;
+                if (link.load) {
-                    } else {
+                    FreebaseProperty property = link.property;
-                        sb.append(", ");
+                    
+                    Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
+                    if (c != null) {
+                        if (first) {
+                            first = false;
+                        } else {
+                            sb.append(", ");
+                        }
+                        sb.append("\"" + property.id + "\": ");
+                        sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
+                    }
+                    
+                    if (child instanceof CellTopicTransposedNode) {
+                        CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
+                        Recon recon = child2.cell.recon;
+                        
+                        if (recon != null &&
+                            (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
+                            
+                            if (firstRecon) {
+                                firstRecon = false;
+                            } else {
+                                sbRecon.append(", ");
+                            }
+                            
+                            sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
+                            
+                            writeRecon(sbRecon, project, 
+                                rowIndices.get(i), child2.cellIndex, child2.cell);
+                        }
                     }
-                    sb.append("\"" + properties.get(i).id + "\": ");
-                    sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
                 }
             }
-            sb.append(" }");
+            sb.append(" }, \"meta\" : { \"recon\" : { ");
+            sb.append(sbRecon.toString());
+            sb.append(" } } }");
             
-            writeLine(subject, predicate, sb, project, subjectRowIndex, subjectCellIndex, subjectCell, -1, -1, null);
+            writeLine(sb.toString());
             
             return null;
         }
@@ -257,7 +319,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
             this.cell = cell;
         }
         
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             String id = null;
             int objectRowIndex = -1;
             int objectCellIndex = -1;
@@ -307,7 +369,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
     }
     
-    protected class CellValueTransposedNode implements WritingTransposedNode {
+    protected class CellValueTransposedNode extends WritingTransposedNode {
         protected JSONObject obj;
         protected CellValueNode node;
         protected int rowIndex;
@@ -321,7 +383,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
             this.cell = cell;
         }
         
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             if (subject != null) {
                 if ("/type/text".equals(node.lang)) {
                     writeLine(subject, predicate, cell.value, node.lang, project, 
@@ -337,7 +399,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
     }
     
-    protected class CellKeyTransposedNode implements WritingTransposedNode {
+    protected class CellKeyTransposedNode extends WritingTransposedNode {
         protected CellKeyNode node;
         protected int rowIndex;
         protected int cellIndex;
@@ -350,7 +412,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
             this.cell = cell;
         }
         
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             writeLine(subject, "key", node.namespace.id + "/" + cell.value, project, 
                 subjectRowIndex, subjectCellIndex, subjectCell, 
                 -1, -1, null);
@@ -366,7 +428,7 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
             this.node = node;
         }
 
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             writeLine(subject, predicate, node.topic.id, project, 
                 subjectRowIndex, subjectCellIndex, subjectCell, 
                 -1, -1, null);
@@ -377,14 +439,14 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
     }
 
-    protected class ValueTransposedNode implements WritingTransposedNode {
+    protected class ValueTransposedNode extends WritingTransposedNode {
         protected ValueNode node;
         
         public ValueTransposedNode(ValueNode node) {
             this.node = node;
         }
 
-        public Object write(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+        public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
             if ("/type/text".equals(node.lang)) {
                 writeLine(subject, predicate, node.value, node.lang, project,
                     subjectRowIndex, subjectCellIndex, subjectCell);
@@ -400,29 +462,48 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
     
     public TransposedNode transposeAnonymousNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             AnonymousNode node, int rowIndex) {
         
+        WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
         WritingTransposedNode tnode = new AnonymousTransposedNode();
         
-        processTransposedNode(tnode, parentNode, property);
+        tnode.load = 
+            (parentNode2 == null || parentNode2.load) &&
+            (link == null || link.load);
+        
+        processTransposedNode(tnode, parentNode, link, rowIndex);
+        
+        tnode.jsonContextNode = addJsonContext(
+            parentNode2 != null ? parentNode2.jsonContextNode : null,
+            link != null ? link.property.id : null,
+            null
+        );
         
         return tnode;
     }
 
     public TransposedNode transposeCellNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
             CellNode node, 
             int rowIndex, 
             Cell cell) {
         
+        WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
+        
         Column column = project.columnModel.getColumnByName(node.columnName);
         int cellIndex = column != null ? column.getCellIndex() : -1;
         
         WritingTransposedNode tnode = null;
         if (node instanceof CellTopicNode) {
-            tnode = new CellTopicTransposedNode((CellTopicNode) node, rowIndex, cellIndex, cell);
+            if (cell.recon != null && 
+                    (cell.recon.judgment == Judgment.Matched ||
+                            cell.recon.judgment == Judgment.New)) {
+                
+                tnode = new CellTopicTransposedNode(
+                    (CellTopicNode) node, rowIndex, cellIndex, cell);
+            }
         } else if (node instanceof CellValueNode) {
             tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
         } else if (node instanceof CellKeyNode) {
@@ -430,31 +511,66 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
         }
         
         if (tnode != null) {
-            processTransposedNode(tnode, parentNode, property);
+            tnode.load = 
+                (parentNode2 == null || parentNode2.load) &&
+                (link == null || link.load);
+            
+            processTransposedNode(tnode, parentNode, link, rowIndex);
+            
+            tnode.jsonContextNode = addJsonContext(
+                parentNode2 != null ? parentNode2.jsonContextNode : null,
+                link != null ? link.property.id : null,
+                cell,
+                rowIndex
+            );
         }
         return tnode;
     }
 
     public TransposedNode transposeTopicNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
-            FreebaseTopicNode node, int rowIndex) {
+            FreebaseTopicNode node,
+            int rowIndex) {
         
+        WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
         WritingTransposedNode tnode = new TopicTransposedNode(node);
         
-        processTransposedNode(tnode, parentNode, property);
+        tnode.load = 
+            (parentNode2 == null || parentNode2.load) &&
+            (link == null || link.load);
+        
+        processTransposedNode(tnode, parentNode, link, rowIndex);
+        
+        tnode.jsonContextNode = addJsonContext(
+            parentNode2 != null ? parentNode2.jsonContextNode : null,
+            link != null ? link.property.id : null,
+            node.topic
+        );
         
         return tnode;
     }
 
     public TransposedNode transposeValueNode(
             TransposedNode parentNode,
-            FreebaseProperty property, 
+            Link link, 
-            ValueNode node, int rowIndex) {
+            ValueNode node,
+            int rowIndex) {
         
+        WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
         WritingTransposedNode tnode = new ValueTransposedNode(node);
         
-        processTransposedNode(tnode, parentNode, property);
+        tnode.load = 
+            (parentNode2 == null || parentNode2.load) &&
+            (link == null || link.load);
+        
+        processTransposedNode(tnode, parentNode, link, rowIndex);
+        
+        tnode.jsonContextNode = addJsonContext(
+            parentNode2 != null ? parentNode2.jsonContextNode : null,
+            link != null ? link.property.id : null,
+            node.value
+        );
         
         return tnode;
     }
@@ -462,23 +578,125 @@ public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory
     protected void processTransposedNode(
         WritingTransposedNode  tnode, 
         TransposedNode         parentNode,
-        FreebaseProperty       property 
+        Link                   link,
+        int                    rowIndex 
     ) {
         if (parentNode != null) {
             if (parentNode instanceof TransposedNodeWithChildren) {
                 TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
+                parentNode2.rowIndices.add(rowIndex);
                 parentNode2.children.add(tnode);
-                parentNode2.properties.add(property);
+                parentNode2.links.add(link);
             }
         } else {
-            addRootNode(tnode);
+            addRootNode(tnode, rowIndex);
         }
     }
     
-    protected void addRootNode(WritingTransposedNode tnode) {
+    protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
+        JSONObject o = new JSONObject();
+        
+        try {
+            if (value instanceof FreebaseTopic) {
+                FreebaseTopic topic = (FreebaseTopic) value;
+                o.put("id", topic.id);
+                o.put("name", topic.name);
+            } else {
+                o.put("v", value);
+            }
+        } catch (JSONException e) {
+            // ignore
+        }
+        
+        connectJsonContext(parent, o, key);
+        return o;
+    }
+    
+    protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
+        JSONObject o = new JSONObject();
+        
+        connectJsonContext(parent, o, key);
+        
+        try {
+            if (cell != null) {
+                o.put("v", cell.value);
+                if (cell.recon != null) {
+                    o.put("recon", "rec" + cell.recon.id);
+                    
+                    if (cell.recon.judgment == Judgment.Matched) {
+                        o.put("id", cell.recon.match.id);
+                        o.put("name", cell.recon.match.name);
+                    }
+                    
+                    // qa:display_context
+                    {
+                        StringBuffer sb2 = new StringBuffer();
+                        
+                        sb2.append("{ \"s\" : \"rec");
+                        sb2.append(Long.toString(cell.recon.id));
+                        sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
+                        sb2.append(Long.toString(contextID));
+                        sb2.append("\", \"meta\" : { \"row\" : ");
+                        sb2.append(Integer.toString(rowIndex));
+                        sb2.append(" } }");
+                        
+                        writeLine(sb2.toString());
+                    }
+                }
+            }
+        } catch (JSONException e) {
+            // ignore
+        }
+        
+        return o;
+    }
+    
+    protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
+        try {
+            if (parent == null) {
+                contextTreeRoot = o;
+            } else {
+                JSONArray a = null;
+                if (parent.has(key)) {
+                    a = parent.getJSONArray(key);
+                } else {
+                    a = new JSONArray();
+                    parent.put(key, a);
+                }
+                
+                a.put(o);
+            }
+        } catch (JSONException e) {
+            // ignore
+        }
+    }
+    
+    protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
         if (lastRootNode != null) {
             lastRootNode.write(null, null, project, -1, -1, null);
+            writeContextTreeNode();
         }
         lastRootNode = tnode;
+        
+        contextTreeRoot = null;
+        contextRowIndex = rowIndex;
+        contextRefCount = 0;
+        contextID++;
+    }
+    
+    protected void writeContextTreeNode() {
+        if (contextTreeRoot != null && contextRefCount > 0) {
+            StringBuffer sb = new StringBuffer();
+            
+            sb.append("{ \"s\" : \"ctx"); 
+            sb.append(Long.toString(contextID)); 
+            sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
+            sb.append(Integer.toString(contextRowIndex));
+            sb.append(", \"data\" : ");
+            sb.append(contextTreeRoot.toString());
+            sb.append(" } } }");
+            
+            writeLine(sb.toString());
+        }
     }
 }