s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

The Freebase functionalities have been extracted out in the last commit. We're removing them from the core module now. This is not a complete checkin. SVN is having some trouble with some directories.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1452 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-10-08 01:54:00 +00:00
parent be17d0b3b4
commit 6ddd945a80
48 changed files with 152 additions and 5200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
main/.classpath
View File

@ -13,10 +13,6 @@
<classpathentry kind="lib" path="webapp/WEB-INF/lib/secondstring-20100303.jar" sourcepath="webapp/WEB-INF/lib-src/secondstring-20100303-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/ant-tools-1.8.0.jar" sourcepath="webapp/WEB-INF/lib-src/ant-tools-1.8.0-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/vicino-1.1.jar" sourcepath="webapp/WEB-INF/lib-src/vicino-1.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/httpcore-4.0.1.jar" sourcepath="webapp/WEB-INF/lib-src/httpcore-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/signpost-core-1.2.1.1.jar" sourcepath="webapp/WEB-INF/lib-src/signpost-core-1.2.1.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.1.jar" sourcepath="webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/opencsv-2.2.jar" sourcepath="tests/java/lib-src/opencsv-2.2-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar" sourcepath="webapp/WEB-INF/lib-src/jcl-over-slf4j-1.5.6-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/slf4j-api-1.5.6.jar" sourcepath="webapp/WEB-INF/lib/slf4j-api-1.5.6.jar"/>
@ -39,5 +35,9 @@
<classpathentry kind="lib" path="tests/server/lib/mockito-all-1.8.4.jar" sourcepath="tests/server/lib-src/mockito-all-1.8.4-sources.jar"/>
<classpathentry kind="lib" path="tests/server/lib/testng-5.12.1.jar" sourcepath="tests/server/lib-src/testng-5.12.1-sources.jar"/>
<classpathentry kind="lib" path="/grefine-server/lib/servlet-api-2.5.jar" sourcepath="/grefine-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/httpcore-4.0.1.jar" sourcepath="webapp/WEB-INF/lib-src/httpcore-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.1.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/signpost-core-1.2.1.1.jar"/>
<classpathentry kind="output" path="webapp/WEB-INF/classes"/>
</classpath>

15
main/src/com/google/refine/RefineServlet.java
View File

@ -262,6 +262,14 @@ public class RefineServlet extends Butterfly {
}
static final private Map<String, String> classMappingsCache = new HashMap<String, String>();
static final private Map<String, Class<?>> classCache = new HashMap<String, Class<?>>();
// TODO(dfhuynh): Temporary solution until we figure out why cross butterfly module class resolution
// doesn't entirely work
static public void cacheClass(Class<?> klass) {
classCache.put(klass.getName(), klass);
}
static public Class<?> getClass(String className) throws ClassNotFoundException {
String toClassName = classMappingsCache.get(className);
if (toClassName == null) {
@ -282,6 +290,11 @@ public class RefineServlet extends Butterfly {
classMappingsCache.put(className, toClassName);
}
return Class.forName(toClassName);
Class<?> klass = classCache.get(toClassName);
if (klass == null) {
klass = Class.forName(toClassName);
classCache.put(toClassName, klass);
}
return klass;
}
}

32
main/src/com/google/refine/commands/column/ExtendDataCommand.java
View File

@ -1,32 +0,0 @@
package com.google.refine.commands.column;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.column.ExtendDataOperation;
import com.google.refine.util.ParsingUtilities;
public class ExtendDataCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project,
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String baseColumnName = request.getParameter("baseColumnName");
int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
String jsonString = request.getParameter("extension");
JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
return new ExtendDataOperation(
engineConfig,
baseColumnName,
extension,
columnInsertIndex
);
}
}

158
main/src/com/google/refine/commands/column/PreviewExtendDataCommand.java
View File

@ -1,158 +0,0 @@
package com.google.refine.commands.column;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.util.FreebaseDataExtensionJob;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.util.FreebaseDataExtensionJob.DataExtension;
public class PreviewExtendDataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String columnName = request.getParameter("columnName");
String rowIndicesString = request.getParameter("rowIndices");
if (rowIndicesString == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }");
return;
}
String jsonString = request.getParameter("extension");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
int length = rowIndices.length();
int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
List<String> topicNames = new ArrayList<String>();
List<String> topicIds = new ArrayList<String>();
Set<String> ids = new HashSet<String>();
for (int i = 0; i < length; i++) {
int rowIndex = rowIndices.getInt(i);
if (rowIndex >= 0 && rowIndex < project.rows.size()) {
Row row = project.rows.get(rowIndex);
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
topicNames.add(cell.recon.match.name);
topicIds.add(cell.recon.match.id);
ids.add(cell.recon.match.id);
} else {
topicNames.add(null);
topicIds.add(null);
ids.add(null);
}
}
}
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("code"); writer.value("ok");
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.key("names");
writer.array();
for (String name : info.names) {
writer.value(name);
}
writer.endArray();
writer.key("path");
writer.array();
for (String id : info.path) {
writer.value(id);
}
writer.endArray();
writer.endObject();
}
writer.endArray();
writer.key("rows");
writer.array();
for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r);
String topicName = topicNames.get(r);
if (id != null && map.containsKey(id)) {
DataExtension ext = map.get(id);
boolean first = true;
if (ext.data.length > 0) {
for (Object[] row : ext.data) {
writer.array();
if (first) {
writer.value(topicName);
first = false;
} else {
writer.value(null);
}
for (Object cell : row) {
if (cell != null && cell instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) cell;
writer.object();
writer.key("id"); writer.value(rc.id);
writer.key("name"); writer.value(rc.name);
writer.endObject();
} else {
writer.value(cell);
}
}
writer.endArray();
}
continue;
}
}
writer.array();
if (id != null) {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(topicName);
writer.endObject();
} else {
writer.value("<not reconciled>");
}
writer.endArray();
}
writer.endArray();
writer.endObject();
} catch (Exception e) {
respondException(response, e);
}
}
}

36
main/src/com/google/refine/commands/freebase/ImportQADataCommand.java
View File

@ -1,36 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.recon.ImportQADataOperation;
import com.google.refine.process.Process;
public class ImportQADataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
AbstractOperation op = new ImportQADataOperation();
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

31
main/src/com/google/refine/commands/freebase/MQLReadCommand.java
View File

@ -1,31 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
import com.google.refine.util.FreebaseUtils;
public class MQLReadCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String query = request.getParameter("query");
String result = FreebaseUtils.mqlread(provider,query);
response.getWriter().write(result);
} catch (Exception e) {
respondException(response, e);
}
}
}

40
main/src/com/google/refine/commands/freebase/MQLWriteCommand.java
View File

@ -1,40 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
import com.google.refine.util.FreebaseUtils;
public class MQLWriteCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
if (access_credentials != null) {
String query = request.getParameter("query");
String result = FreebaseUtils.mqlwrite(access_credentials, provider, query);
response.getWriter().write(result);
} else {
respond(response, "401 Unauthorized", "You don't have the right credentials");
}
} catch (Exception e) {
respondException(response, e);
}
}
}

71
main/src/com/google/refine/commands/freebase/PreviewProtographCommand.java
View File

@ -1,71 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
import com.google.refine.protograph.Protograph;
import com.google.refine.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.protograph.transpose.Transposer;
import com.google.refine.protograph.transpose.TripleLoaderTransposedNodeFactory;
import com.google.refine.util.ParsingUtilities;
public class PreviewProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
FilteredRows filteredRows = engine.getAllFilteredRows();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
StringBuffer sb = new StringBuffer(2048);
sb.append("{ ");
{
StringWriter stringWriter = new StringWriter();
TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append("\"tripleloader\" : ");
sb.append(JSONObject.quote(stringWriter.toString()));
}
{
StringWriter stringWriter = new StringWriter();
MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append(", \"mqllike\" : ");
sb.append(stringWriter.toString());
}
sb.append(" }");
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

40
main/src/com/google/refine/commands/freebase/SaveProtographCommand.java
View File

@ -1,40 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.SaveProtographOperation;
import com.google.refine.process.Process;
import com.google.refine.protograph.Protograph;
import com.google.refine.util.ParsingUtilities;
public class SaveProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
AbstractOperation op = new SaveProtographOperation(protograph);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
}
}
}

89
main/src/com/google/refine/commands/freebase/UploadDataCommand.java
View File

@ -1,89 +0,0 @@
package com.google.refine.commands.freebase;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.exporters.ProtographTransposeExporter.TripleLoaderExporter;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.FreebaseUtils;
import com.google.refine.util.ParsingUtilities;
public class UploadDataCommand extends Command {
final static public String s_dataLoadJobIDPref = "freebase.load.jobID";
final static public String s_dataLoadJobNamePref = "freebase.load.jobName";
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore();
TripleLoaderExporter exporter = new TripleLoaderExporter();
StringWriter triples = new StringWriter(10 * 1024 * 1024);
exporter.export(project, new Properties(), engine, triples);
String source_name = request.getParameter("source_name");
String source_id = request.getParameter("source_id");
String qa = request.getParameter("qa");
String mdo_id = null;
preferenceStore.put(s_dataLoadJobNamePref, source_name);
try {
Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref);
if (jobID != null) {
URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID);
String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
if (!s.equals("null")) {
mdo_id = s;
}
}
} catch (Exception e) {
// ignore
}
String uploadResponse = FreebaseUtils.uploadTriples(
request, qa, source_name, source_id, mdo_id, triples.toString()
);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
try {
JSONObject obj = new JSONObject(uploadResponse);
if (obj.has("result") && !obj.isNull("result")) {
JSONObject result = obj.getJSONObject("result");
if (result.has("job_id") && !result.isNull("job_id")) {
Integer jobID = result.getInt("job_id");
project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
}
}
response.getWriter().write(uploadResponse);
} catch (JSONException e) {
respond(response,"500 Error", uploadResponse);
}
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

2
main/src/com/google/refine/commands/freebase/GuessTypesOfColumnCommand.java → main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -1,4 +1,4 @@
package com.google.refine.commands.freebase;
package com.google.refine.commands.recon;
import java.io.DataOutputStream;
import java.io.IOException;

7
main/src/com/google/refine/exporters/ExporterRegistry.java
View File

@ -3,10 +3,6 @@ package com.google.refine.exporters;
import java.util.HashMap;
import java.util.Map;
import com.google.refine.exporters.ProtographTransposeExporter.MqlwriteLikeExporter;
import com.google.refine.exporters.ProtographTransposeExporter.TripleLoaderExporter;
abstract public class ExporterRegistry {
static final private Map<String, Exporter> s_formatToExporter = new HashMap<String, Exporter>();
@ -16,9 +12,6 @@ abstract public class ExporterRegistry {
s_formatToExporter.put("csv", new CsvExporter());
s_formatToExporter.put("template", new TemplatingExporter());
s_formatToExporter.put("tripleloader", new TripleLoaderExporter());
s_formatToExporter.put("mqlwrite", new MqlwriteLikeExporter());
}
static public void registerExporter(String format, Exporter exporter) {

74
main/src/com/google/refine/exporters/ProtographTransposeExporter.java
View File

@ -1,74 +0,0 @@
package com.google.refine.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Properties;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
import com.google.refine.protograph.Protograph;
import com.google.refine.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.protograph.transpose.TransposedNodeFactory;
import com.google.refine.protograph.transpose.Transposer;
import com.google.refine.protograph.transpose.TripleLoaderTransposedNodeFactory;
abstract public class ProtographTransposeExporter implements Exporter {
final protected String _contentType;
public ProtographTransposeExporter(String contentType) {
_contentType = contentType;
}
public String getContentType() {
return "application/x-unknown";
}
public boolean takeWriter() {
return true;
}
public void export(Project project, Properties options, Engine engine,
OutputStream outputStream) throws IOException {
throw new RuntimeException("Not implemented");
}
public void export(Project project, Properties options, Engine engine,
Writer writer) throws IOException {
Protograph protograph = (Protograph) project.overlayModels.get("freebaseProtograph");
if (protograph != null) {
TransposedNodeFactory nodeFactory = createNodeFactory(project, writer);
Transposer.transpose(project, engine.getAllFilteredRows(),
protograph, protograph.getRootNode(0), nodeFactory, -1);
nodeFactory.flush();
}
}
abstract protected TransposedNodeFactory createNodeFactory(Project project, Writer writer);
static public class TripleLoaderExporter extends ProtographTransposeExporter {
public TripleLoaderExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new TripleLoaderTransposedNodeFactory(project, writer);
}
}
static public class MqlwriteLikeExporter extends ProtographTransposeExporter {
public MqlwriteLikeExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new MqlwriteLikeTransposedNodeFactory(writer);
}
}
}

5
main/src/com/google/refine/model/Project.java
View File

@ -24,7 +24,6 @@ import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.history.History;
import com.google.refine.process.ProcessManager;
import com.google.refine.protograph.Protograph;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
@ -36,10 +35,6 @@ public class Project {
s_overlayModelClasses.put(modelName, klass);
}
static {
registerOverlayModel("freebaseProtograph", Protograph.class);
}
final public long id;
final public List<Row> rows = new ArrayList<Row>();

431
main/src/com/google/refine/model/changes/DataExtensionChange.java
View File

@ -1,431 +0,0 @@
package com.google.refine.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Serializable;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.history.Change;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.DataExtensionReconConfig;
import com.google.refine.protograph.FreebaseType;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
import com.google.refine.util.FreebaseDataExtensionJob.DataExtension;
public class DataExtensionChange implements Change {
final protected String _baseColumnName;
final protected int _columnInsertIndex;
final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes;
final protected List<Integer> _rowIndices;
final protected List<DataExtension> _dataExtensions;
protected long _historyEntryID;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
public DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
long historyEntryID
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_historyEntryID = historyEntryID;
}
protected DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_firstNewCellIndex = firstNewCellIndex;
_oldRows = oldRows;
_newRows = newRows;
}
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_oldRows = new ArrayList<Row>(project.rows);
_newRows = new ArrayList<Row>(project.rows.size());
int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
int index = 0;
int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) {
if (r2 < project.rows.size()) {
Row oldRow2 = project.rows.get(r2);
if (oldRow2.isCellBlank(cellIndex) &&
oldRow2.isCellBlank(keyCellIndex)) {
Row newRow = oldRow2.dup();
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
r2++;
continue;
}
}
Row newRow = new Row(cellIndex + _columnNames.size());
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
}
}
project.rows.clear();
project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
try {
project.columnModel.addColumn(_columnInsertIndex + i, column, true);
// the column might have been renamed to avoid collision
_columnNames.set(i, column.getName());
} catch (ModelException e) {
// won't get here since we set the avoid collision flag
}
}
project.update();
}
}
protected void extendRow(
Row row,
DataExtension dataExtension,
int extensionRowIndex,
Map<String, Recon> reconMap
) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.id)) {
recon = reconMap.get(rc.id);
} else {
recon = Recon.makeFreebaseRecon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "mql";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.id, recon);
}
cell = new Cell(rc.name, recon);
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
public void revert(Project project) {
synchronized (project) {
project.rows.clear();
project.rows.addAll(_oldRows);
for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnInsertIndex);
}
project.update();
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) {
writer.write(name); writer.write('\n');
}
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (FreebaseType type : _columnTypes) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
type.write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
writer.write('\n');
}
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
if (dataExtension == null) {
writer.write('\n');
continue;
}
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
for (Row row : _oldRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null;
int columnInsertIndex = -1;
List<String> columnNames = null;
List<FreebaseType> columnTypes = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Row> oldRows = null;
List<Row> newRows = null;
int firstNewCellIndex = -1;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) {
baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value);
rowIndices = new ArrayList<Integer>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
rowIndices.add(Integer.parseInt(line));
}
}
} else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value);
columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
columnNames.add(line);
}
}
} else if ("columnTypeCount".equals(field)) {
int count = Integer.parseInt(value);
columnTypes = new ArrayList<FreebaseType>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
}
} else if ("dataExtensionCount".equals(field)) {
int count = Integer.parseInt(value);
dataExtensions = new ArrayList<DataExtension>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line == null) continue;
if (line.length() == 0) {
dataExtensions.add(null);
continue;
}
int rowCount = Integer.parseInt(line);
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
}
} else if ("oldRowCount".equals(field)) {
int count = Integer.parseInt(value);
oldRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line, pool));
}
}
} else if ("newRowCount".equals(field)) {
int count = Integer.parseInt(value);
newRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line, pool));
}
}
}
}
DataExtensionChange change = new DataExtensionChange(
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows
);
return change;
}
}

63
main/src/com/google/refine/model/recon/DataExtensionReconConfig.java
View File

@ -1,63 +0,0 @@
package com.google.refine.model.recon;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.protograph.FreebaseType;
public class DataExtensionReconConfig extends StrictReconConfig {
final public FreebaseType type;
private final static String WARN = "Not implemented";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject type = obj.getJSONObject("type");
return new DataExtensionReconConfig(
new FreebaseType(
type.getString("id"),
type.getString("name")
)
);
}
public DataExtensionReconConfig(FreebaseType type) {
this.type = type;
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
throw new RuntimeException(WARN);
}
@Override
public int getBatchSize() {
throw new RuntimeException(WARN);
}
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("mode"); writer.value("extend");
writer.key("type"); type.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
throw new RuntimeException(WARN);
}
@Override
public String getBriefDescription(Project project, String columnName) {
throw new RuntimeException(WARN);
}
}

174
main/src/com/google/refine/model/recon/GuidBasedReconConfig.java
View File

@ -1,174 +0,0 @@
package com.google.refine.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.util.ParsingUtilities;
public class GuidBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new GuidBasedReconConfig();
}
public GuidBasedReconConfig() {
}
static protected class GuidBasedReconJob extends ReconJob {
String guid;
public int getKey() {
return guid.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
GuidBasedReconJob job = new GuidBasedReconJob();
String s = cell.value.toString();
if (s.startsWith("/guid/")) {
s = "#" + s.substring(6);
} else if (!s.startsWith("#")) {
s = "#" + s;
}
job.guid = s;
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("guid|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((GuidBasedReconJob) job).guid);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String guid = result.getString("guid");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String guid = ((GuidBasedReconJob) jobs.get(i)).guid;
Recon recon = guidToRecon.get(guid);
recons.add(recon);
}
return recons;
}
}

179
main/src/com/google/refine/model/recon/IdBasedReconConfig.java
View File

@ -1,179 +0,0 @@
package com.google.refine.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.util.ParsingUtilities;
public class IdBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new IdBasedReconConfig();
}
public IdBasedReconConfig() {
}
static protected class IdBasedReconJob extends ReconJob {
String id;
public int getKey() {
return id.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
IdBasedReconJob job = new IdBasedReconJob();
String s = cell.value.toString();
if (!s.startsWith("/")) {
if (s.startsWith("92")) {
s = "/guid/" + s;
} else if (!s.contains("/")){
s = "/en/" + s;
} else {
s = "/" + s;
}
}
job.id = s;
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> idToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("id|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((IdBasedReconJob) job).id);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String id = result.getString("id");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String id = ((IdBasedReconJob) jobs.get(i)).id;
Recon recon = idToRecon.get(id);
recons.add(recon);
}
return recons;
}
}

193
main/src/com/google/refine/model/recon/KeyBasedReconConfig.java
View File

@ -1,193 +0,0 @@
package com.google.refine.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.protograph.FreebaseTopic;
import com.google.refine.util.ParsingUtilities;
public class KeyBasedReconConfig extends StrictReconConfig {
final public FreebaseTopic namespace;
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject ns = obj.getJSONObject("namespace");
return new KeyBasedReconConfig(
new FreebaseTopic(
ns.getString("id"),
ns.getString("name")
)
);
}
public KeyBasedReconConfig(FreebaseTopic namespace) {
this.namespace = namespace;
}
static protected class KeyBasedReconJob extends ReconJob {
String key;
public int getKey() {
return key.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
KeyBasedReconJob job = new KeyBasedReconJob();
job.key = cell.value.toString().replace(' ', '_');
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("key");
writer.key("namespace"); namespace.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("key");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("namespace");
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(namespace.id);
jsonWriter.endObject();
jsonWriter.key("value"); jsonWriter.value(null);
jsonWriter.key("value|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((KeyBasedReconJob) job).key);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String key = ((KeyBasedReconJob) jobs.get(i)).key;
Recon recon = keyToRecon.get(key);
recons.add(recon);
}
return recons;
}
}

57
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -1,7 +1,11 @@
package com.google.refine.model.recon;
import java.io.Writer;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
@ -14,17 +18,52 @@ import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import edu.mit.simile.butterfly.ButterflyModule;
abstract public class ReconConfig implements Jsonizable {
static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass =
new HashMap<String, List<Class<? extends ReconConfig>>>();
static final public Map<Class<? extends ReconConfig>, String> s_opClassToName =
new HashMap<Class<? extends ReconConfig>, String>();
static public void registerReconConfig(ButterflyModule module, String name, Class<? extends ReconConfig> klass) {
String key = module.getName() + "/" + name;
s_opClassToName.put(klass, key);
List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(key);
if (classes == null) {
classes = new LinkedList<Class<? extends ReconConfig>>();
s_opNameToClass.put(key, classes);
}
classes.add(klass);
}
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String mode = obj.getString("mode");
if ("standard-service".equals(mode) ||
"heuristic".equals(mode) // legacy
) {
return StandardReconConfig.reconstruct(obj);
} else if ("strict".equals(mode)) {
return StrictReconConfig.reconstruct(obj);
} else if ("extend".equals(mode)) {
return DataExtensionReconConfig.reconstruct(obj);
try {
String mode = obj.getString("mode");
// Backward compatibility
if ("extend".equals(mode) || "strict".equals("mode")) {
mode = "freebase/" + mode;
} else if ("heuristic".equals(mode)) {
mode = "core/standard-service"; // legacy
} else if (!mode.contains("/")) {
mode = "core/" + mode;
}
List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(mode);
if (classes != null && classes.size() > 0) {
Class<? extends ReconConfig> klass = classes.get(classes.size() - 1);
Method reconstruct = klass.getMethod("reconstruct", JSONObject.class);
if (reconstruct != null) {
return (ReconConfig) reconstruct.invoke(null, obj);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

36
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -27,7 +27,6 @@ import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.RecordModel.RowDependency;
import com.google.refine.protograph.FreebaseProperty;
import com.google.refine.util.ParsingUtilities;
public class StandardReconConfig extends ReconConfig {
@ -35,11 +34,13 @@ public class StandardReconConfig extends ReconConfig {
static public class ColumnDetail {
final public String columnName;
final public FreebaseProperty property;
final public String propertyName;
final public String propertyID;
public ColumnDetail(String columnName, FreebaseProperty property) {
public ColumnDetail(String columnName, String propertyName, String propertyID) {
this.columnName = columnName;
this.property = property;
this.propertyName = propertyName;
this.propertyID = propertyID;
}
}
@ -52,15 +53,21 @@ public class StandardReconConfig extends ReconConfig {
columnDetails = new ArrayList<ColumnDetail>(l);
for (int i = 0; i < l; i++) {
JSONObject o = columnDetailsA.getJSONObject(i);
JSONObject p = o.getJSONObject("property");
columnDetails.add(new ColumnDetail(
o.getString("column"),
new FreebaseProperty(
p.getString("id"),
p.getString("name")
)
));
if (o.has("property")) { // legacy
JSONObject p = o.getJSONObject("property");
columnDetails.add(new ColumnDetail(
o.getString("column"),
p.has("name") ? p.getString("name") : null,
p.has("id") ? p.getString("id") : null
));
} else {
columnDetails.add(new ColumnDetail(
o.getString("column"),
o.has("propertyName") ? o.getString("propertyName") : null,
o.has("propertyID") ? o.getString("propertyID") : null
));
}
}
} else {
columnDetails = new ArrayList<ColumnDetail>();
@ -140,7 +147,8 @@ public class StandardReconConfig extends ReconConfig {
for (ColumnDetail c : columnDetails) {
writer.object();
writer.key("column"); writer.value(c.columnName);
writer.key("property"); c.property.write(writer, options);
writer.key("propertyName"); writer.value(c.propertyName);
writer.key("propertyID"); writer.value(c.propertyID);
writer.endObject();
}
writer.endArray();
@ -198,7 +206,7 @@ public class StandardReconConfig extends ReconConfig {
if (cell2 != null && ExpressionUtils.isNonBlankData(cell2.value)) {
jsonWriter.object();
jsonWriter.key("pid"); jsonWriter.value(c.property.id);
jsonWriter.key("pid"); jsonWriter.value(c.propertyID);
jsonWriter.key("v");
if (cell2.recon != null && cell2.recon.match != null) {
jsonWriter.object();

19
main/src/com/google/refine/model/recon/StrictReconConfig.java
View File

@ -1,19 +0,0 @@
package com.google.refine.model.recon;
import org.json.JSONObject;
abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match");
if ("key".equals(match)) {
return KeyBasedReconConfig.reconstruct(obj);
} else if ("id".equals(match)) {
return IdBasedReconConfig.reconstruct(obj);
} else if ("guid".equals(match)) {
return GuidBasedReconConfig.reconstruct(obj);
}
return null;
}
}

21
main/src/com/google/refine/oauth/FreebaseProvider.java
View File

@ -1,21 +0,0 @@
package com.google.refine.oauth;
public class FreebaseProvider extends Provider {
public FreebaseProvider(String host) {
super(host);
}
public String getRequestTokenServiceURL() {
return "https://" + host + "/api/oauth/request_token";
}
public String getAccessTokenServiceURL() {
return "https://" + host + "/api/oauth/access_token";
}
public String getUserAuthorizationURL() {
return "https://" + host + "/signin/app";
}
}

69
main/src/com/google/refine/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
View File

@ -1,69 +0,0 @@
package com.google.refine.oauth;
import java.io.IOException;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer {
final static Logger logger = LoggerFactory.getLogger("oauth");
private static final long serialVersionUID = -4139931605235255279L;
private static final int SOCKET_TIMEOUT = 3000;
private static final int CONNECTION_TIMEOUT = 3000;
private static final String TIMER_URL = "http://refinery.freebaseapps.com/time";
public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) {
super(consumerKey, consumerSecret);
}
/**
* It might be that the user's computer's clock is not synchronized enough with the Freebase servers
* and this might result in Freebase thinking that it was under a replay attack.
* To avoid this problem we get the timestamp directly from acre that we know is synchronized.
*
* NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same
* could be said if we used an NTP client.
*/
protected String generateTimestamp() {
long time = -1;
try {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT);
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT);
HttpClient httpClient = new DefaultHttpClient(httpParams);
HttpGet httpget = new HttpGet(TIMER_URL);
HttpResponse response = httpClient.execute(httpget);
HttpEntity entity = response.getEntity();
if (entity != null) {
time = Long.parseLong(EntityUtils.toString(entity),10);
logger.debug("Got remote timestamp {}", time);
}
} catch (IOException e) {
logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e);
}
if (time == -1) {
time = System.currentTimeMillis();
}
return Long.toString(time / 1000L);
}
}

1
main/src/com/google/refine/oauth/GoogleProvider.java
View File

@ -13,5 +13,4 @@ public class GoogleProvider extends Provider {
public String getUserAuthorizationURL() {
return "https://www.google.com/accounts/OAuthAuthorizeToken";
}
}

16
main/src/com/google/refine/oauth/OAuthUtilities.java
View File

@ -10,20 +10,14 @@ import oauth.signpost.OAuthProvider;
import oauth.signpost.commonshttp.CommonsHttpOAuthProvider;
import oauth.signpost.http.HttpParameters;
import com.google.refine.util.FreebaseUtils;
public class OAuthUtilities {
static final private Map<String,Provider> providers = new HashMap<String,Provider>();
static final private Map<String,String[]> infos = new HashMap<String,String[]>();
static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"};
static {
Provider freebase = new FreebaseProvider(FreebaseUtils.FREEBASE_HOST);
providers.put(freebase.getHost(), freebase);
infos.put(freebase.getHost(), FREEBASE_OAUTH_INFO);
static final public void registerOAuthProvider(Provider provider, String[] oauthInfo) {
providers.put(provider.getHost(), provider);
infos.put(provider.getHost(), oauthInfo);
}
public static Provider getProvider(String name) {
@ -43,7 +37,7 @@ public class OAuthUtilities {
if (provider == null) throw new RuntimeException("Provider can't be null");
String[] consumer_info = infos.get(provider.getHost());
if (consumer_info == null) throw new RuntimeException("Can't find secrets for provider '" + provider.getHost() + "'");
OAuthConsumer oauthConsumer = new FreebaseTimeCommonsHttpOAuthConsumer(consumer_info[0],consumer_info[1]);
OAuthConsumer oauthConsumer = provider.createConsumer(consumer_info[0],consumer_info[1]);
HttpParameters params = new HttpParameters();
params.put("realm", provider.getHost());
oauthConsumer.setAdditionalParameters(params);

7
main/src/com/google/refine/oauth/Provider.java
View File

@ -1,5 +1,8 @@
package com.google.refine.oauth;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
public abstract class Provider {
@ -19,4 +22,8 @@ public abstract class Provider {
abstract public String getRequestTokenServiceURL();
abstract public String getAccessTokenServiceURL();
abstract public String getUserAuthorizationURL();
public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) {
return new CommonsHttpOAuthConsumer(consumerKey, consumerSecret);
}
}

120
main/src/com/google/refine/operations/SaveProtographOperation.java
View File

@ -1,120 +0,0 @@
package com.google.refine.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.protograph.Protograph;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveProtographOperation extends AbstractOperation {
final protected Protograph _protograph;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new SaveProtographOperation(
Protograph.reconstruct(obj.getJSONObject("protograph"))
);
}
public SaveProtographOperation(Protograph protograph) {
_protograph = protograph;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription());
writer.key("protograph"); _protograph.write(writer, options);
writer.endObject();
}
protected String getBriefDescription() {
return "Save schema alignment skeleton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Change change = new ProtographChange(_protograph);
return new HistoryEntry(historyEntryID, project, getBriefDescription(), SaveProtographOperation.this, change);
}
static public class ProtographChange implements Change {
final protected Protograph _newProtograph;
protected Protograph _oldProtograph;
public ProtographChange(Protograph protograph) {
_newProtograph = protograph;
}
public void apply(Project project) {
synchronized (project) {
_oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph");
project.overlayModels.put("freebaseProtograph", _newProtograph);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldProtograph == null) {
project.overlayModels.remove("freebaseProtograph");
} else {
project.overlayModels.put("freebaseProtograph", _oldProtograph);
}
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
Protograph oldProtograph = null;
Protograph newProtograph = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldProtograph".equals(field) && value.length() > 0) {
oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newProtograph".equals(field) && value.length() > 0) {
newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
ProtographChange change = new ProtographChange(newProtograph);
change._oldProtograph = oldProtograph;
return change;
}
static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
if (p != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
p.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

275
main/src/com/google/refine/operations/column/ExtendDataOperation.java
View File

@ -1,275 +0,0 @@
package com.google.refine.operations.column;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellAtRow;
import com.google.refine.model.changes.DataExtensionChange;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
import com.google.refine.protograph.FreebaseType;
import com.google.refine.util.FreebaseDataExtensionJob;
import com.google.refine.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.util.FreebaseDataExtensionJob.DataExtension;
public class ExtendDataOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected JSONObject _extension;
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ExtendDataOperation(
engineConfig,
obj.getString("baseColumnName"),
obj.getJSONObject("extension"),
obj.getInt("columnInsertIndex")
);
}
public ExtendDataOperation(
JSONObject engineConfig,
String baseColumnName,
JSONObject extension,
int columnInsertIndex
) {
super(engineConfig);
_baseColumnName = baseColumnName;
_extension = extension;
_columnInsertIndex = columnInsertIndex;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
writer.key("baseColumnName"); writer.value(_baseColumnName);
writer.key("extension"); writer.value(_extension);
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + _baseColumnName;
}
protected String createDescription(Column column, List<CellAtRow> cellsAtRows) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + column.getName() +
" by filling " + cellsAtRows.size();
}
public Process createProcess(Project project, Properties options) throws Exception {
return new ExtendDataProcess(
project,
getEngineConfig(),
getBriefDescription(null)
);
}
public class ExtendDataProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected JSONObject _engineConfig;
final protected long _historyEntryID;
protected int _cellIndex;
protected FreebaseDataExtensionJob _job;
public ExtendDataProcess(
Project project,
JSONObject engineConfig,
String description
) throws JSONException {
super(description);
_project = project;
_engineConfig = engineConfig;
_historyEntryID = HistoryEntry.allocateID();
_job = new FreebaseDataExtensionJob(_extension);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(hashCode());
writer.key("description"); writer.value(_description);
writer.key("immediate"); writer.value(false);
writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done"));
writer.key("progress"); writer.value(_progress);
writer.endObject();
}
protected Runnable getRunnable() {
return this;
}
protected void populateRowsWithMatches(List<Integer> rowIndices) throws Exception {
Engine engine = new Engine(_project);
engine.initializeFromJSON(_engineConfig);
Column column = _project.columnModel.getColumnByName(_baseColumnName);
if (column == null) {
throw new Exception("No column named " + _baseColumnName);
}
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
return false;
}
}.init(rowIndices));
}
protected int extendRows(
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
Map<String, ReconCandidate> reconCandidateMap
) {
Set<String> ids = new HashSet<String>();
int end;
for (end = from; end < limit && ids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
ids.add(cell.recon.match.id);
}
Map<String, DataExtension> map = null;
try {
map = _job.extend(ids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.id;
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
return end;
}
public void run() {
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try {
populateRowsWithMatches(rowIndices);
} catch (Exception e2) {
// TODO : Not sure what to do here?
e2.printStackTrace();
}
int start = 0;
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
_progress = end * 100 / rowIndices.size();
try {
Thread.sleep(200);
} catch (InterruptedException e) {
if (_canceled) {
break;
}
}
}
if (!_canceled) {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) {
columnTypes.add(info.expectedType);
}
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
ExtendDataOperation.this,
new DataExtensionChange(
_baseColumnName,
_columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
_historyEntryID)
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
}
}
}

106
main/src/com/google/refine/operations/recon/ImportQADataOperation.java
View File

@ -1,106 +0,0 @@
package com.google.refine.operations.recon;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.freebase.UploadDataCommand;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassReconChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
public class ImportQADataOperation extends AbstractOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new ImportQADataOperation();
}
public ImportQADataOperation() {
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Import QA DAta";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref);
if (jobID == null) {
throw new InternalError("Project is not associated with any data loading job.");
}
Map<Long, String> reconIDToResult = new HashMap<Long, String>();
URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(30000); // 30 seconds
LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream()));
try {
String line;
while ((line = reader.readLine()) != null) {
JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line);
long reconID = Long.parseLong(obj.getString("recon_id").substring(3));
reconIDToResult.put(reconID, obj.getString("result"));
}
} finally {
reader.close();
}
Map<Long, Recon> oldRecons = new HashMap<Long, Recon>();
Map<Long, Recon> newRecons = new HashMap<Long, Recon>();
for (int r = 0; r < project.rows.size(); r++) {
Row row = project.rows.get(r);
for (int c = 0; c < row.cells.size(); c++) {
Cell cell = row.cells.get(c);
if (cell != null && cell.recon != null) {
Recon oldRecon = cell.recon;
if (reconIDToResult.containsKey(oldRecon.id)) {
Recon newRecon = oldRecon.dup();
newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id));
reconIDToResult.remove(oldRecon.id);
oldRecons.put(oldRecon.id, oldRecon);
newRecons.put(oldRecon.id, newRecon);
}
}
}
}
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(project),
this,
new MassReconChange(newRecons, oldRecons)
);
}
}

415
main/src/com/google/refine/util/FreebaseDataExtensionJob.java
View File

@ -1,415 +0,0 @@
/**
*
*/
package com.google.refine.util;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.ReconCandidate;
import com.google.refine.protograph.FreebaseType;
public class FreebaseDataExtensionJob {
static public class DataExtension {
final public Object[][] data;
public DataExtension(Object[][] data) {
this.data = data;
}
}
static public class ColumnInfo {
final public List<String> names;
final public List<String> path;
final public FreebaseType expectedType;
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names;
this.path = path;
this.expectedType = expectedType;
}
}
final public JSONObject extension;
final public int columnCount;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
this.extension = obj;
this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ?
countColumns(obj.getJSONArray("properties"), columns, new ArrayList<String>(), new ArrayList<String>()) : 0;
}
public Map<String, FreebaseDataExtensionJob.DataExtension> extend(
Set<String> ids,
Map<String, ReconCandidate> reconCandidateMap
) throws Exception {
StringWriter writer = new StringWriter();
formulateQuery(ids, extension, writer);
String query = writer.toString();
InputStream is = doMqlRead(query);
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
if (o.has("result")) {
JSONArray a = o.getJSONArray("result");
int l = a.length();
for (int i = 0; i < l; i++) {
JSONObject o2 = a.getJSONObject(i);
String id = o2.getString("id");
FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
if (ext != null) {
map.put(id, ext);
}
}
}
return map;
} finally {
is.close();
}
}
protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
List<Object[]> rows = new ArrayList<Object[]>();
collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
Object[][] data = new Object[rows.size()][columnCount];
rows.toArray(data);
return new DataExtension(data);
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
Object value,
Map<String, ReconCandidate> reconCandidateMap
) {
while (row >= rows.size()) {
rows.add(new Object[columnCount]);
}
rows.get(row)[col] = value;
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String id = obj.getString("id");
ReconCandidate rc;
if (reconCandidateMap.containsKey(id)) {
rc = reconCandidateMap.get(id);
} else {
rc = new ReconCandidate(
obj.getString("id"),
obj.getString("name"),
JSONUtilities.getStringArray(obj, "type"),
100
);
reconCandidateMap.put(id, rc);
}
storeCell(rows, row, col, rc, reconCandidateMap);
}
protected int[] collectResult(
List<Object[]> rows,
JSONObject extNode,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String propertyID = extNode.getString("id");
String expectedTypeID = extNode.getJSONObject("expected").getString("id");
JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ?
resultNode.getJSONArray(propertyID) : null;
if (expectedTypeID.startsWith("/type/")) {
if (a != null) {
int l = a.length();
for (int r = 0; r < l; r++) {
Object o = a.isNull(r) ? null : a.get(r);
if (o instanceof Serializable) {
storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap);
}
}
}
// note that we still take up a column even if we don't have any data
return new int[] { startRowIndex, startColumnIndex + 1 };
} else {
boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
if (a != null && a.length() > 0) {
int maxColIndex = startColumnIndex;
int l = a.length();
for (int r = 0; r < l; r++) {
Object v = a.isNull(r) ? null : a.get(r);
JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null;
int startColumnIndex2 = startColumnIndex;
int startRowIndex2 = startRowIndex;
if (isOwnColumn) {
if (o != null) {
storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap);
} else {
storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap);
}
}
if (hasSubProperties && o != null) {
int[] rowcol = collectResult(
rows,
extNode.getJSONArray("properties"),
o,
startRowIndex,
startColumnIndex2,
reconCandidateMap
);
startRowIndex2 = rowcol[0];
startColumnIndex2 = rowcol[1];
}
startRowIndex = startRowIndex2;
maxColIndex = Math.max(maxColIndex, startColumnIndex2);
}
return new int[] { startRowIndex, maxColIndex };
} else {
return new int[] {
startRowIndex,
startColumnIndex + countColumns(extNode, null, new ArrayList<String>(), new ArrayList<String>())
};
}
}
}
protected int[] collectResult(
List<Object[]> rows,
JSONArray subProperties,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
int maxStartRowIndex = startRowIndex;
int k = subProperties.length();
for (int c = 0; c < k; c++) {
int[] rowcol = collectResult(
rows,
subProperties.getJSONObject(c),
resultNode,
startRowIndex,
startColumnIndex,
reconCandidateMap
);
maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]);
startColumnIndex = rowcol[1];
}
return new int[] { maxStartRowIndex, startColumnIndex };
}
static protected InputStream doMqlRead(String query) throws IOException {
URL url = new URL("http://api.freebase.com/api/service/mqlread");
URLConnection connection = url.openConnection();
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
try {
String body = "extended=1&query=" + ParsingUtilities.encode(query);
dos.writeBytes(body);
} finally {
dos.flush();
dos.close();
}
connection.connect();
return connection.getInputStream();
}
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("id|=");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
}
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
String propertyID = node.getString("id");
String expectedTypeID = node.getJSONObject("expected").getString("id");
writer.key(propertyID);
writer.array();
{
if (!expectedTypeID.startsWith("/type/")) { // not literal
writer.object();
writer.key("optional"); writer.value(true);
boolean hasLimit = false;
if (node.has("constraints") && !node.isNull("constraints")) {
JSONObject constraints = node.getJSONObject("constraints");
String[] names = JSONObject.getNames(constraints);
for (String name : names) {
Object value = constraints.get(name);
if (name.equals("limit")) {
hasLimit = true;
}
if (!name.contains(":") &&
!name.equals("limit") &&
!name.equals("optional") &&
!name.equals("count") &&
!name.equals("estimate-count") &&
!name.equals("sort") &&
!name.equals("return")) {
if (name.startsWith("!")) {
name = "!c:" + name.substring(1);
} else {
name = "c:" + name;
}
}
writer.key(name);
writer.value(value);
}
}
if (!hasLimit) {
writer.key("limit"); writer.value(10);
}
{
boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
writer.key("name"); writer.value(null);
writer.key("id"); writer.value(null);
writer.key("type"); writer.array(); writer.endArray();
}
if (hasSubProperties) {
formulateQueryNode(node.getJSONArray("properties"), writer);
}
}
writer.endObject();
}
}
writer.endArray();
}
static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
int l = propertiesA.length();
for (int i = 0; i < l; i++) {
formulateQueryNode(propertiesA.getJSONObject(i), writer);
}
}
static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
String name = obj.getString("name");
List<String> names2 = null;
List<String> path2 = null;
if (columns != null) {
names2 = new ArrayList<String>(names);
names2.add(name);
path2 = new ArrayList<String>(path);
path2.add(obj.getString("id"));
}
if (obj.has("properties") && !obj.isNull("properties")) {
boolean included = (obj.has("included") && obj.getBoolean("included"));
if (included && columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return (included ? 1 : 0) +
countColumns(obj.getJSONArray("properties"), columns, names2, path2);
} else {
if (columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return 1;
}
}
static protected int countColumns(JSONArray a, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
int c = 0;
int l = a.length();
for (int i = 0; i < l; i++) {
c += countColumns(a.getJSONObject(i), columns, names, path);
}
return c;
}
}

46
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -53,12 +53,11 @@ function registerCommands() {
RS.registerCommand(module, "transpose-rows-into-columns", new Packages.com.google.refine.commands.cell.TransposeRowsIntoColumnsCommand());
RS.registerCommand(module, "add-column", new Packages.com.google.refine.commands.column.AddColumnCommand());
RS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.refine.commands.column.AddColumnByFetchingURLsCommand());
RS.registerCommand(module, "remove-column", new Packages.com.google.refine.commands.column.RemoveColumnCommand());
RS.registerCommand(module, "rename-column", new Packages.com.google.refine.commands.column.RenameColumnCommand());
RS.registerCommand(module, "move-column", new Packages.com.google.refine.commands.column.MoveColumnCommand());
RS.registerCommand(module, "split-column", new Packages.com.google.refine.commands.column.SplitColumnCommand());
RS.registerCommand(module, "extend-data", new Packages.com.google.refine.commands.column.ExtendDataCommand());
RS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.refine.commands.column.AddColumnByFetchingURLsCommand());
RS.registerCommand(module, "reorder-columns", new Packages.com.google.refine.commands.column.ReorderColumnsCommand());
RS.registerCommand(module, "denormalize", new Packages.com.google.refine.commands.row.DenormalizeCommand());
@ -70,33 +69,17 @@ function registerCommands() {
RS.registerCommand(module, "recon-match-specific-topic-to-cells", new Packages.com.google.refine.commands.recon.ReconMatchSpecificTopicCommand());
RS.registerCommand(module, "recon-judge-one-cell", new Packages.com.google.refine.commands.recon.ReconJudgeOneCellCommand());
RS.registerCommand(module, "recon-judge-similar-cells", new Packages.com.google.refine.commands.recon.ReconJudgeSimilarCellsCommand());
RS.registerCommand(module, "guess-types-of-column", new Packages.com.google.refine.commands.recon.GuessTypesOfColumnCommand());
RS.registerCommand(module, "annotate-one-row", new Packages.com.google.refine.commands.row.AnnotateOneRowCommand());
RS.registerCommand(module, "annotate-rows", new Packages.com.google.refine.commands.row.AnnotateRowsCommand());
RS.registerCommand(module, "remove-rows", new Packages.com.google.refine.commands.row.RemoveRowsCommand());
RS.registerCommand(module, "reorder-rows", new Packages.com.google.refine.commands.row.ReorderRowsCommand());
RS.registerCommand(module, "save-protograph", new Packages.com.google.refine.commands.freebase.SaveProtographCommand());
RS.registerCommand(module, "get-expression-language-info", new Packages.com.google.refine.commands.expr.GetExpressionLanguageInfoCommand());
RS.registerCommand(module, "get-expression-history", new Packages.com.google.refine.commands.expr.GetExpressionHistoryCommand());
RS.registerCommand(module, "log-expression", new Packages.com.google.refine.commands.expr.LogExpressionCommand());
RS.registerCommand(module, "preview-expression", new Packages.com.google.refine.commands.expr.PreviewExpressionCommand());
RS.registerCommand(module, "preview-extend-data", new Packages.com.google.refine.commands.column.PreviewExtendDataCommand());
RS.registerCommand(module, "preview-protograph", new Packages.com.google.refine.commands.freebase.PreviewProtographCommand());
RS.registerCommand(module, "guess-types-of-column", new Packages.com.google.refine.commands.freebase.GuessTypesOfColumnCommand());
RS.registerCommand(module, "check-authorization", new Packages.com.google.refine.commands.auth.CheckAuthorizationCommand());
RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand());
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand());
RS.registerCommand(module, "user-badges", new Packages.com.google.refine.commands.auth.GetUserBadgesCommand());
RS.registerCommand(module, "upload-data", new Packages.com.google.refine.commands.freebase.UploadDataCommand());
RS.registerCommand(module, "import-qa-data", new Packages.com.google.refine.commands.freebase.ImportQADataCommand());
RS.registerCommand(module, "mqlread", new Packages.com.google.refine.commands.freebase.MQLReadCommand());
RS.registerCommand(module, "mqlwrite", new Packages.com.google.refine.commands.freebase.MQLWriteCommand());
RS.registerCommand(module, "get-preference", new Packages.com.google.refine.commands.GetPreferenceCommand());
RS.registerCommand(module, "get-all-preferences", new Packages.com.google.refine.commands.GetAllPreferencesCommand());
@ -122,7 +105,6 @@ function registerOperations() {
OR.registerOperation(module, "column-rename", Packages.com.google.refine.operations.column.ColumnRenameOperation);
OR.registerOperation(module, "column-move", Packages.com.google.refine.operations.column.ColumnMoveOperation);
OR.registerOperation(module, "column-split", Packages.com.google.refine.operations.column.ColumnSplitOperation);
OR.registerOperation(module, "extend-data", Packages.com.google.refine.operations.column.ExtendDataOperation);
OR.registerOperation(module, "column-addition-by-fetching-urls", Packages.com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation);
OR.registerOperation(module, "column-reorder", Packages.com.google.refine.operations.column.ColumnReorderOperation);
@ -137,11 +119,6 @@ function registerOperations() {
OR.registerOperation(module, "recon-discard-judgments", Packages.com.google.refine.operations.recon.ReconDiscardJudgmentsOperation);
OR.registerOperation(module, "recon-match-specific-topic-to-cells", Packages.com.google.refine.operations.recon.ReconMatchSpecificTopicOperation);
OR.registerOperation(module, "recon-judge-similar-cells", Packages.com.google.refine.operations.recon.ReconJudgeSimilarCellsOperation);
OR.registerOperation(module, "import-qa-data", Packages.com.google.refine.operations.recon.ImportQADataOperation);
// for backward compatibility
OR.registerOperation(module, "save-protograph", Packages.com.google.refine.operations.SaveProtographOperation);
OR.registerOperation(module, "save-schema-alignment-skeleton", Packages.com.google.refine.operations.SaveProtographOperation);
}
/*
@ -153,6 +130,9 @@ function init() {
registerCommands();
registerOperations();
var RC = Packages.com.google.refine.model.recon.ReconConfig;
RC.registerReconConfig(module, "standard-service", Packages.com.google.refine.model.recon.StandardReconConfig);
ClientSideResourceManager.addPaths(
"index/scripts",
module,
@ -198,8 +178,6 @@ function init() {
"scripts/util/menu.js",
"scripts/util/dialog.js",
"scripts/util/dom.js",
"scripts/util/sign.js",
"scripts/util/freebase.js",
"scripts/util/custom-suggest.js",
"scripts/widgets/history-widget.js",
@ -233,16 +211,10 @@ function init() {
"scripts/reconciliation/standard-service-panel.js",
"scripts/dialogs/expression-preview-dialog.js",
"scripts/dialogs/freebase-loading-dialog.js",
"scripts/dialogs/clustering-dialog.js",
"scripts/dialogs/scatterplot-dialog.js",
"scripts/dialogs/extend-data-preview-dialog.js",
"scripts/dialogs/templating-exporter-dialog.js",
"scripts/dialogs/column-reordering-dialog.js",
"scripts/protograph/schema-alignment.js",
"scripts/protograph/schema-alignment-ui-node.js",
"scripts/protograph/schema-alignment-ui-link.js"
"scripts/dialogs/column-reordering-dialog.js"
]
);
@ -276,14 +248,10 @@ function init() {
"styles/dialogs/expression-preview-dialog.less",
"styles/dialogs/clustering-dialog.less",
"styles/dialogs/scatterplot-dialog.less",
"styles/dialogs/freebase-loading-dialog.less",
"styles/dialogs/extend-data-preview-dialog.less",
"styles/dialogs/column-reordering-dialog.less",
"styles/reconciliation/recon-dialog.less",
"styles/reconciliation/standard-service-panel.less",
"styles/protograph/schema-alignment-dialog.less"
"styles/reconciliation/standard-service-panel.less"
]
);

26
main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.html
View File

@ -1,26 +0,0 @@
<div class="dialog-frame extend-data-preview-dialog" style="width: 1000px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div class="grid-layout layout-normal layout-full"><table rows="4">
<tr>
<td width="300" height="1">Add Property</td>
<td height="1">Preview</td>
<td height="1" width="1%"><button bind="resetButton">Reset</button></td>
</tr>
<tr>
<td style="vertical-align: top;" height="1"><div class="input-container"><input bind="addPropertyInput" /></div></td>
<td style="vertical-align: top;" rowspan="3" colspan="2"><div class="preview-container" bind="previewContainer"></div></td>
</tr>
<tr>
<td height="1">Suggested Properties</td>
</tr>
<tr>
<td><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>
</tr>
</table></div>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>
<button bind="cancelButton">Cancel</button>
</div>
</div>

363
main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.js
View File

@ -1,363 +0,0 @@
function ExtendDataPreviewDialog(column, columnIndex, rowIndices, onDone) {
this._column = column;
this._columnIndex = columnIndex;
this._rowIndices = rowIndices;
this._onDone = onDone;
this._extension = { properties: [] };
var self = this;
this._dialog = $(DOM.loadHTML("core", "scripts/dialogs/extend-data-preview-dialog.html"));
this._elmts = DOM.bind(this._dialog);
this._elmts.dialogHeader.text("Add Columns from Freebase Based on Column " + column.name);
this._elmts.resetButton.click(function() {
self._extension.properties = [];
self._update();
});
this._elmts.okButton.click(function() {
if (self._extension.properties.length === 0) {
alert("Please add some properties first.");
} else {
DialogSystem.dismissUntil(self._level - 1);
self._onDone(self._extension);
}
});
this._elmts.cancelButton.click(function() {
DialogSystem.dismissUntil(self._level - 1);
});
var dismissBusy = DialogSystem.showBusy();
var type = "reconConfig" in column && "type" in column.reconConfig ? column.reconConfig.type.id : "/common/topic";
ExtendDataPreviewDialog.getAllProperties(type, function(properties) {
dismissBusy();
self._show(properties);
});
}
ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
var done = false;
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?type=" + typeID + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
var allProperties = [];
for (var i = 0; i < data.properties.length; i++) {
var property = data.properties[i];
var property2 = {
id: property.id,
name: property.name
};
if ("id2" in property) {
property2.expected = property.schema2;
property2.properties = [{
id: property.id2,
name: property.name2,
expected: property.expects
}];
} else {
property2.expected = property.expects;
}
allProperties.push(property2);
}
allProperties.sort(function(a, b) { return a.name.localeCompare(b.name); });
onDone(allProperties);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
ExtendDataPreviewDialog.prototype._show = function(properties) {
this._level = DialogSystem.showDialog(this._dialog);
var n = this._elmts.suggestedPropertyContainer.offset().top +
this._elmts.suggestedPropertyContainer.outerHeight(true) -
this._elmts.addPropertyInput.offset().top;
this._elmts.previewContainer.height(Math.floor(n));
var self = this;
var container = this._elmts.suggestedPropertyContainer;
var renderSuggestedProperty = function(property) {
var label = ("properties" in property) ? (property.name + " &raquo; " + property.properties[0].name) : property.name;
var div = $('<div>').addClass("suggested-property").appendTo(container);
$('<a>')
.attr("href", "javascript:{}")
.html(label)
.appendTo(div)
.click(function() {
self._addProperty(property);
});
};
for (var i = 0; i < properties.length; i++) {
renderSuggestedProperty(properties[i]);
}
var suggestConfig = {
type: '/type/property'
};
if ("reconConfig" in this._column && "type" in this._column.reconConfig) {
suggestConfig.ac_param = { schema: this._column.reconConfig.type.id };
}
this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
var expected = data.expected_type;
self._addProperty({
id : data.id,
name: data.name,
expected: {
id: expected.id,
name: expected.name
}
});
});
};
ExtendDataPreviewDialog.prototype._update = function() {
this._elmts.previewContainer.empty().text("Querying Freebase ...");
var self = this;
var params = {
project: theProject.id,
columnName: this._column.name
};
$.post(
"/command/core/preview-extend-data?" + $.param(params),
{
rowIndices: JSON.stringify(this._rowIndices),
extension: JSON.stringify(this._extension)
},
function(data) {
self._renderPreview(data);
},
"json"
);
};
ExtendDataPreviewDialog.prototype._addProperty = function(p) {
var addSeveralToList = function(properties, oldProperties) {
for (var i = 0; i < properties.length; i++) {
addToList(properties[i], oldProperties);
}
};
var addToList = function(property, oldProperties) {
for (var i = 0; i < oldProperties.length; i++) {
var oldProperty = oldProperties[i];
if (oldProperty.id == property.id) {
if ("included" in property) {
oldProperty.included = "included" in oldProperty ?
(oldProperty.included || property.included) :
property.included;
}
if ("properties" in property) {
if ("properties" in oldProperty) {
addSeveralToList(property.properties, oldProperty.properties);
} else {
oldProperty.properties = property.properties;
}
}
return;
}
}
oldProperties.push(property);
};
addToList(p, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
var self = this;
var container = this._elmts.previewContainer.empty();
if (data.code == "error") {
container.text("Error.");
return;
}
var table = $('<table>')[0];
var trHead = table.insertRow(table.rows.length);
$('<th>').appendTo(trHead).text(this._column.name);
var renderColumnHeader = function(column) {
var th = $('<th>').appendTo(trHead);
$('<span>').html(column.names.join(" &raquo; ")).appendTo(th);
$('<br>').appendTo(th);
$('<a href="javascript:{}"></a>')
.text("remove")
.addClass("action")
.attr("title", "Remove this column")
.click(function() {
self._removeProperty(column.path);
}).appendTo(th);
$('<a href="javascript:{}"></a>')
.text("constrain")
.addClass("action")
.attr("title", "Add constraints to this column")
.click(function() {
self._constrainProperty(column.path);
}).appendTo(th);
};
for (var c = 0; c < data.columns.length; c++) {
renderColumnHeader(data.columns[c]);
}
for (var r = 0; r < data.rows.length; r++) {
var tr = table.insertRow(table.rows.length);
var row = data.rows[r];
for (var c = 0; c < row.length; c++) {
var td = tr.insertCell(tr.cells.length);
var cell = row[c];
if (cell !== null) {
if ($.isPlainObject(cell)) {
$('<a>').attr("href", "http://www.freebase.com/view" + cell.id).text(cell.name).appendTo(td);
} else {
$('<span>').text(cell).appendTo(td);
}
}
}
}
container.append(table);
};
ExtendDataPreviewDialog.prototype._removeProperty = function(path) {
var removeFromList = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
if ("included" in property) {
delete property.included;
}
} else if ("properties" in property && property.properties.length > 0) {
removeFromList(path, index + 1, property.properties);
}
if (!("properties" in property) || property.properties.length === 0) {
properties.splice(i, 1);
}
return;
}
}
};
removeFromList(path, 0, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._findProperty = function(path) {
var find = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
return property;
} else if ("properties" in property && property.properties.length > 0) {
return find(path, index + 1, property.properties);
}
break;
}
}
return null;
};
return find(path, 0, this._extension.properties);
};
ExtendDataPreviewDialog.prototype._constrainProperty = function(path) {
var self = this;
var property = this._findProperty(path);
var frame = DialogSystem.createDialog();
frame.width("500px");
var header = $('<div></div>').addClass("dialog-header").text("Constrain " + path.join(" > ")).appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
body.html(
'<div class="grid-layout layout-normal layout-full"><table>' +
'<tr><td>' +
'Enter MQL query constraints as JSON' +
'</td></tr>' +
'<tr><td>' +
'<textarea style="width: 100%; height: 300px; font-family: monospace;" bind="textarea"></textarea>' +
'</td></tr>' +
'</table></div>'
);
var bodyElmts = DOM.bind(body);
if ("constraints" in property) {
bodyElmts.textarea[0].value = JSON.stringify(property.constraints, null, 2);
} else {
bodyElmts.textarea[0].value = JSON.stringify({ "limit" : 10 }, null, 2);
}
footer.html(
'<button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>' +
'<button bind="cancelButton">Cancel</button>'
);
var footerElmts = DOM.bind(footer);
var level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(level - 1);
};
footerElmts.cancelButton.click(dismiss);
footerElmts.okButton.click(function() {
try {
var o = JSON.parse(bodyElmts.textarea[0].value);
if (o === undefined) {
alert("Please ensure that the JSON you enter is valid.");
return;
}
if ($.isArray(o) && o.length == 1) {
o = o[0];
}
if (!$.isPlainObject(o)) {
alert("The JSON you enter must be an object, that is, it is of this form { ... }.");
return;
}
property.constraints = o;
dismiss();
self._update();
} catch (e) {
//console.log(e);
}
});
bodyElmts.textarea.focus();
};

19
main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html
View File

@ -1,19 +0,0 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader">Load Data into Freebase</div>
<div class="dialog-body" bind="dialogBody">
</div>
<div class="dialog-footer" bind="dialogFooter">
<table width="100%"><tr>
<td bind="left" style="text-align: left" width="40%" nowrap="true">
<button bind="cancelButton" id="freebase-loading-cancel">Cancel</button>
</td>
<td bind="center" style="text-align: center" width="20%" nowrap="true">
<div bind="authorization" class="freebase-loading-authorization"></div>
</td>
<td bind="right" style="text-align: right" width="40%" nowrap="true">
<span bind="qaCheckboxContainer">Ready to perform QA on this data? <input bind="qaCheckbox" type="checkbox" name="qa" value="qa"></span>
<button bind="loadButton" id="freebase-loading-load" disabled="disabled">Load to Sandbox</button>
</td>
</tr></table>
</div>
</div>

225
main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js
View File

@ -1,225 +0,0 @@
function FreebaseLoadingDialog() {
this._createDialog();
this._signedin = false;
}
FreebaseLoadingDialog.prototype._createDialog = function() {
var self = this;
var dialog = $(DOM.loadHTML("core", "scripts/dialogs/freebase-loading-dialog.html"));
this._elmts = DOM.bind(dialog);
this._elmts.cancelButton.click(function() { self._dismiss(); });
var provider = "www.freebase.com";
var authorization = this._elmts.authorization;
var loadButton = this._elmts.loadButton;
var check_authorization = function(cont) {
$.get("/command/core/check-authorization/" + provider, function(data) {
if ("status" in data && data.code == "/api/status/ok") {
authorization.html('Signed in as: <a target="_new" href="http://www.freebase.com/view/user/' + data.username + '">' + data.username + '</a> | <a href="javascript:{}" bind="signout">Sign Out</a>').show();
DOM.bind(authorization).signout.click(function() {
self._signedin = false;
loadButton.attr("disabled","disabled");
$("#freebase-loading-graph-selector-freebase").attr("disabled","disabled").button("refresh");
Sign.signout(check_authorization,provider);
});
loadButton.unbind().click(function() {
self._load();
});
self._signedin = true;
$("#freebase-loading-source-name").keyup();
if (typeof cont == "function") cont(data);
} else {
authorization.html('<a href="javascript:{}" bind="signin">Sign into Freebase</a> to enable loading').show();
DOM.bind(authorization).signin.click(function() {
Sign.signin(function() {
check_authorization(cont);
},provider);
});
}
},"json");
};
var make_topic = function(new_topic_id, topic_type, cont) {
var mql_query = [{
"create": "unless_exists",
"name": new_topic_id,
"a:type": topic_type,
"b:type": "/common/topic",
"id": null,
"guid": null
}];
$.post("/command/core/mqlwrite/" + provider,
{ "query" : JSON.stringify(mql_query) },
function(data) {
if ("status" in data && data.code == "/api/status/ok") {
self._elmts.source_id.val(data.result[0].id);
if (typeof cont == "function") cont();
} else {
self._show_error("Error creating new topic", data);
}
},
"json"
);
};
var show_triples = function(cont) {
$.post(
"/command/core/preview-protograph?" + $.param({ project: theProject.id }),
{
protograph: JSON.stringify(theProject.overlayModels.freebaseProtograph || {}),
engine: JSON.stringify(ui.browsingEngine.getJSON())
},
function(data) {
var body = self._elmts.dialogBody;
if ("tripleloader" in data) {
body.html(
'<div class="freebase-loading-tripleloader-info"><table><tr>' +
'<td><div>Name this data load &not; <sup style="color: red">required</sup></div>' +
'<input type="text" size="40" id="freebase-loading-source-name" bind="source_name"></td>' +
'<td><div>Source ID &not; <sup style="color: #888">optional</sup></div>' +
'<input type="text" size="60" id="freebase-loading-source-id" bind="source_id"></td>' +
'</tr></table></div>' +
'<div class="freebase-loading-tripleloader-data">' + data.tripleloader + '</div>'
);
self._elmts = DOM.bind(dialog);
self._elmts.source_name.keyup(function() {
if (self._signedin && $(this).val() != "") {
loadButton.removeAttr("disabled");
} else {
loadButton.attr("disabled","disabled");
}
});
self._elmts.source_id.suggest({
"type": "/dataworld/information_source",
"suggest_new": "Click here to add a new information source"
}).bind("fb-select", function(e, data) {
self._elmts.source_id.val(data.id);
}).bind("fb-select-new", function(e, val) {
make_topic(val, "/dataworld/information_source");
});
$.getJSON(
"/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobName" }),
null,
function(data) {
if (data.value != null) {
self._elmts.source_name[0].value = data.value;
}
}
);
if (typeof cont == "function") cont();
} else {
body.html(
'<div class="freebase-loading-tripleloader-message">'+
'<h2>This dataset has no triples</h2>' +
'<p>Have you aligned it with the Freebase schemas yet?</p>' +
'</div>'
);
self._elmts = DOM.bind(dialog);
self._end();
}
self._level = DialogSystem.showDialog(dialog);
},
"json"
);
};
show_triples(check_authorization);
};
FreebaseLoadingDialog.prototype._load = function() {
var self = this;
var qa = self._elmts.qaCheckbox.is(':checked');
var get_refinery_url = function(url) {
return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0];
};
var doLoad = function() {
var dismissBusy = DialogSystem.showBusy();
$.post("/command/core/upload-data",
{
project: theProject.id,
"qa" : qa,
"engine" : JSON.stringify(ui.browsingEngine.getJSON()),
"source_name" : self._elmts.source_name.val(),
"source_id" : self._elmts.source_id.val()
},
function(data) {
dismissBusy();
var body = self._elmts.dialogBody;
if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) {
body.html(
'<div class="freebase-loading-tripleloader-message">' +
'<h2><span>' + data.result.added + '</span> triples successfully scheduled for loading</h2>' +
'<h4>Follow the loading progress in the <a href="' + get_refinery_url(data.result.status_url) + '" target="_new">Freebase Refinery</a></h4>' +
'</div>'
);
self._end();
} else {
self._show_error("Error loading data",data);
}
},
"json"
);
};
if (qa) {
var dialog = $(
'<div id="freebase-confirmation-dialog" title="Are you sure?">' +
'<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be tested for upload into <b>Freebase</b>?</td></tr></table>' +
'</div>'
).dialog({
resizable: false,
width: 400,
height: 230,
modal: true,
buttons: {
'yes, I\'m sure': function() {
$(this).dialog('close');
doLoad();
},
'hmm, not really': function() {
$(this).dialog('close');
}
}
});
} else {
doLoad();
}
};
FreebaseLoadingDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
FreebaseLoadingDialog.prototype._show_error = function(msg, error) {
console.log(error);
var self = this;
var body = self._elmts.dialogBody;
body.html(
'<div class="freebase-loading-tripleloader-message">' +
'<h2>' + msg + '</h2>' +
(('message' in error) ? '<p>' + error.message + '</p>' : '<pre>' + JSON.stringify(error, null, 2) + '</pre>') +
(('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</pre>' : "") +
'</div>'
);
this._end();
};
FreebaseLoadingDialog.prototype._end = function() {
var self = this;
self._elmts.loadButton.text("Close").removeAttr("disabled").unbind().click(function() {
self._dismiss();
});
self._elmts.cancelButton.hide();
self._elmts.qaCheckboxContainer.hide();
self._elmts.authorization.hide();
};

83
main/webapp/modules/core/scripts/project/extension-bar.js
View File

@ -4,46 +4,14 @@ function ExtensionBar(div) {
}
ExtensionBar.MenuItems = [
{
"id" : "freebase",
"label" : "Freebase",
"submenu" : [
{
"id" : "freebase/schema-alignment",
label: "Edit Schema Aligment Skeleton ...",
click: function() { ExtensionBar.handlers.editSchemaAlignment(false); }
},
{
"id" : "freebase/reset-schema-alignment",
label: "Reset Schema Alignment Skeleton ...",
click: function() { ExtensionBar.handlers.editSchemaAlignment(true); }
},
{},
{
"id" : "freebase/load-info-freebase",
label: "Load into Freebase ...",
click: function() { ExtensionBar.handlers.loadIntoFreebase(); }
},
{
"id" : "freebase/browse-load",
label: "Browse to Data Load ...",
click: function() { ExtensionBar.handlers.browseToDataLoad(); }
},
{
"id" : "freebase/import-qa-data",
label: "Import QA Data",
click: function() { ExtensionBar.handlers.importQAData(); }
}
]
}
];
ExtensionBar.addExtensionMenu = function(what) {
ExtensionBar.appendTo(ExtensionBar.MenuItems, [], what);
MenuSystem.appendTo(ExtensionBar.MenuItems, [], what);
}
ExtensionBar.appendTo = function(path, what) {
ExtensionBar.appendTo(ExtensionBar.MenuItems, path, what);
MenuSystem.appendTo(ExtensionBar.MenuItems, path, what);
};
ExtensionBar.insertBefore = function(path, what) {
@ -99,50 +67,3 @@ ExtensionBar.handlers.openWorkspaceDir = function() {
}
});
};
ExtensionBar.handlers.editSchemaAlignment = function(reset) {
new SchemaAlignmentDialog(
reset ? null : theProject.overlayModels.freebaseProtograph, function(newProtograph) {});
};
ExtensionBar.handlers.loadIntoFreebase = function() {
new FreebaseLoadingDialog();
};
ExtensionBar.handlers.browseToDataLoad = function() {
// The form has to be created as part of the click handler. If you create it
// inside the getJSON success handler, it won't work.
var form = document.createElement("form");
$(form)
.css("display", "none")
.attr("method", "GET")
.attr("target", "dataload");
document.body.appendChild(form);
var w = window.open("about:blank", "dataload");
$.getJSON(
"/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobID" }),
null,
function(data) {
if (data.value == null) {
alert("You have not tried to load the data in this project into Freebase yet.");
} else {
$(form).attr("action", "http://refinery.freebaseapps.com/load/" + data.value);
form.submit();
w.focus();
}
document.body.removeChild(form);
}
);
};
ExtensionBar.handlers.importQAData = function() {
Refine.postCoreProcess(
"import-qa-data",
{},
{},
{ cellsChanged: true }
);
};

348
main/webapp/modules/core/scripts/protograph/schema-alignment-ui-link.js
View File

@ -1,348 +0,0 @@
SchemaAlignmentDialog.UILink = function(dialog, link, table, options, parentUINode) {
this._dialog = dialog;
this._link = link;
this._options = options;
this._parentUINode = parentUINode;
// Make sure target node is there
this._link.target = this._link.target || { nodeType: "cell-as-value" }
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-link-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-link-toggle").attr("width", "1%").addClass("padded");
$(this._tdDetails).addClass("schema-alignment-link-details").attr("width", "90%");
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
var self = this;
var show = function() {
if (self._options.expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._options.expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._options.expanded = !self._options.expanded;
$(this).attr("src", self._options.expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
this._renderMain();
this._renderDetails();
};
SchemaAlignmentDialog.UILink.prototype._renderMain = function() {
$(this._tdMain).empty();
var label = this._link.property !== null ?
(this._link.property.id + ((this._link.condition) ? " [?]" : "")) :
"property?";
var self = this;
$('<img />')
.attr("title", "remove property")
.attr("src", "images/close.png")
.css("cursor", "pointer")
.prependTo(this._tdMain)
.click(function() {
window.setTimeout(function() {
self._parentUINode.removeLink(self);
self._tr.parentNode.removeChild(self._tr);
self._dialog.preview();
}, 100);
});
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-link-tag")
.html(label)
.appendTo(this._tdMain)
.click(function(evt) {
self._startEditProperty(this);
});
$('<img />').attr("src", "images/arrow-start.png").prependTo(a);
$('<img />').attr("src", "images/arrow-end.png").appendTo(a);
};
SchemaAlignmentDialog.UILink.prototype._renderDetails = function() {
if (this._targetUI) {
this._targetUI.dispose();
}
if (this._tableDetails) {
this._tableDetails.remove();
}
this._tableDetails = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv);
this._targetUI = new SchemaAlignmentDialog.UINode(
this._dialog,
this._link.target,
this._tableDetails[0],
{ expanded: "links" in this._link.target && this._link.target.links.length > 0 });
};
SchemaAlignmentDialog.UILink.prototype._startEditProperty = function(elmt) {
var sourceTypeID = this._parentUINode.getExpectedType();
var targetNode = this._targetUI._node;
var targetTypeID = "type" in targetNode && targetNode.type !== null ? targetNode.type.id : null;
var targetTypeName = "columnNames" in targetNode ? targetNode.columnNames[0] : null;
if (sourceTypeID !== null) {
var self = this;
var dismissBusy = DialogSystem.showBusy();
var instanceCount = 0;
var outgoing = [];
var incoming = [];
var onDone = function(properties) {
dismissBusy();
self._showPropertySuggestPopup(
elmt,
properties
);
};
SchemaAlignmentDialog.UILink._getPropertiesOfType(
sourceTypeID,
targetTypeID,
targetTypeName,
onDone
);
} else {
this._showPropertySuggestPopup(elmt, []);
}
};
SchemaAlignmentDialog.UILink._getPropertiesOfType = function(typeID, targetTypeID, targetTypeName, onDone) {
var done = false;
var params = {
"type" : typeID
};
if (targetTypeID !== null) {
params.expects = targetTypeID;
} else if (targetTypeName !== null) {
params.expects = targetTypeName;
}
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?" + $.param(params) + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
onDone(data.properties || []);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
SchemaAlignmentDialog.UILink.prototype._showPropertySuggestPopup = function(elmt, suggestions) {
self = this;
var menu = MenuSystem.createMenu().width("350px");
var commitProperty = function(p) {
window.setTimeout(function() { MenuSystem.dismissAll(); }, 100);
if ("id2" in p) {
// self._targetUI.dispose();
self._link.property = {
id: p.id,
name: p.name
};
self._link.target = {
nodeType: "anonymous",
links: [{
property: {
id: p.id2,
name: p.name2
},
target: self._link.target
}]
};
self._renderDetails();
} else {
self._link.property = {
id: p.id,
name: p.name
};
}
var conditionColumnName = conditionalSelect[0].value;
if (conditionColumnName != "") {
self._link.condition = { columnName: conditionColumnName };
} else {
delete self._link.condition;
}
self._configureTarget();
};
var divConditional = $('<div>')
.addClass("schema-alignment-link-menu-section")
.html("Assert link when 'true' is found in column<br/>").appendTo(menu);
var conditionalSelect = $('<select>').appendTo(divConditional);
$('<option>')
.text("(always assert)")
.attr("value", "")
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
for (var c = 0; c < theProject.columnModel.columns.length; c++) {
var column = theProject.columnModel.columns[c];
var option = $('<option>')
.text(column.name)
.attr("value", column.name)
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
if ((self._link.condition) && column.name == self._link.condition.columnName) {
option.attr("selected", "true");
}
}
var divSearch;
if (suggestions.length > 0) {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section")
.html('<div>Search for a property or pick one below</div>').appendTo(menu);
var createSuggestion = function(suggestion) {
var menuItem = MenuSystem.createMenuItem().appendTo(menu);
$('<span>')
.text(suggestion.name)
.attr("title", suggestion.id)
.appendTo(menuItem);
if ("name2" in suggestion) {
$('<span>').html(" &raquo; ").appendTo(menuItem);
$('<span>')
.text(suggestion.name2)
.attr("title", suggestion.id2)
.appendTo(menuItem);
}
menuItem.click(function() {
commitProperty(suggestion);
});
};
for (var i = 0; i < suggestions.length && i < 10; i++) {
createSuggestion(suggestions[i]);
}
} else {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section-last")
.html('<div>Search for a property</div>').appendTo(menu);
}
var input = $('<input />').appendTo($('<div>').appendTo(divSearch));
MenuSystem.showMenu(menu, function(){});
MenuSystem.positionMenuAboveBelow(menu, $(elmt));
var suggestOptions = {
type : '/type/property'
};
var sourceTypeID = this._parentUINode.getExpectedType();
if (sourceTypeID !== null) {
suggestOptions.ac_param = { schema: sourceTypeID };
}
input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
input[0].focus();
};
SchemaAlignmentDialog.UILink.prototype.getJSON = function() {
if ("property" in this._link && this._link.property !== null &&
"target" in this._link && this._link.target !== null) {
var targetJSON = this._targetUI.getJSON();
if (targetJSON !== null) {
var json = {
property: cloneDeep(this._link.property),
target: targetJSON
};
if (this._link.condition) {
json.condition = cloneDeep(this._link.condition);
}
return json;
}
}
return null;
};
SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
var self = this;
var dismissBusy = DialogSystem.showBusy();
$.getJSON(
"http://api.freebase.com/api/service/mqlread?query=" + JSON.stringify({
query: {
"id" : this._link.property.id,
"type" : "/type/property",
"expected_type" : {
"id" : null,
"name" : null,
"/freebase/type_hints/mediator" : null
}
}
}) + "&callback=?",
null,
function(o) {
dismissBusy();
if ("result" in o) {
var expected_type = o.result.expected_type;
self._link.target.type = {
id: expected_type.id,
name: expected_type.name
};
if (expected_type["/freebase/type_hints/mediator"] === true) {
self._link.target.nodeType = "anonymous";
} else if (expected_type.id == "/type/key") {
self._link.target.nodeType = "cell-as-key";
} else if (expected_type.id.match(/^\/type\//)) {
self._link.target.nodeType = "cell-as-value";
} else if (!("topic" in self._link.target)) {
self._link.target.nodeType = "cell-as-topic";
self._link.target.createForNoReconMatch = true;
}
self._targetUI.render();
}
self._renderMain();
self._dialog.preview();
},
"jsonp"
);
};

732
main/webapp/modules/core/scripts/protograph/schema-alignment-ui-node.js
View File

@ -1,732 +0,0 @@
SchemaAlignmentDialog.UINode = function(dialog, node, table, options) {
this._dialog = dialog;
this._node = node;
this._options = options;
if ("columnName" in this._node) {
this._node.columnNames = [ this._node.columnName ];
delete this._node.columnName;
}
this._linkUIs = [];
this._detailsRendered = false;
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide();
$(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide();
this._renderMain();
this._expanded = options.expanded;
if (this._isExpandable()) {
this._showExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.dispose = function() {
// nothing for now
};
SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) {
for (var i = this._linkUIs.length - 1; i >= 0; i--) {
if (this._linkUIs[i] === linkUI) {
this._linkUIs.splice(i, 1);
this._node.links.splice(i, 1);
break;
}
}
};
SchemaAlignmentDialog.UINode.prototype._isExpandable = function() {
return this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "anonymous" ||
this._node.nodeType == "topic";
};
SchemaAlignmentDialog.UINode.prototype.render = function() {
this._renderMain();
if (this._isExpandable()) {
this._showExpandable();
} else {
this._hideExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() {
if ("type" in this._node) {
return this._node.type.id;
}
return null;
};
SchemaAlignmentDialog.UINode.prototype._renderMain = function() {
$(this._tdMain).empty();
var self = this;
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-node-tag")
.appendTo(this._tdMain)
.click(function(evt) {
self._showNodeConfigDialog();
});
if (this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "cell-as-value" ||
this._node.nodeType == "cell-as-key") {
if ("columnNames" in this._node) {
for (var c = 0; c < this._node.columnNames.length; c++) {
if (c > 0) {
$('<span>').text(", ").appendTo(a);
}
$('<span>')
.text(this._node.columnNames[c])
.addClass("schema-alignment-node-column")
.appendTo(a);
}
$('<span>').text(this._node.columnNames.length > 1 ? " cells" : " cell").appendTo(a);
} else {
a.html(this._options.mustBeCellTopic ? "Which column?" : "Configure...");
}
} else if (this._node.nodeType == "topic") {
if ("topic" in this._node) {
a.html(this._node.topic.name);
} else if ("id" in this._node) {
a.html(this._node.topic.id);
} else {
a.html("Which topic?");
}
} else if (this._node.nodeType == "value") {
if ("value" in this._node) {
a.html(this._node.value);
} else {
a.html("What value?");
}
} else if (this._node.nodeType == "anonymous") {
a.html("(anonymous)");
}
};
SchemaAlignmentDialog.UINode.prototype._showExpandable = function() {
$(this._tdToggle).show();
$(this._tdDetails).show();
if (this._detailsRendered) {
return;
}
this._detailsRendered = true;
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
this._renderDetails();
var self = this;
var show = function() {
if (self._expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._expanded = !self._expanded;
$(this).attr("src", self._expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
};
SchemaAlignmentDialog.UINode.prototype._hideExpandable = function() {
$(this._tdToggle).hide();
$(this._tdDetails).hide();
};
SchemaAlignmentDialog.UINode.prototype._renderDetails = function() {
var self = this;
this._tableLinks = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv)[0];
if ("links" in this._node && this._node.links !== null) {
for (var i = 0; i < this._node.links.length; i++) {
this._linkUIs.push(new SchemaAlignmentDialog.UILink(
this._dialog,
this._node.links[i],
this._tableLinks,
{ expanded: true },
this
));
}
}
var divFooter = $('<div></div>').addClass("padded").appendTo(this._expandedDetailDiv);
$('<a href="javascript:{}"></a>')
.addClass("action")
.text("add property")
.appendTo(divFooter)
.click(function() {
var newLink = {
property: null,
target: {
nodeType: "cell-as-value"
}
};
self._linkUIs.push(new SchemaAlignmentDialog.UILink(
self._dialog,
newLink,
self._tableLinks,
{
expanded: true,
mustBeCellTopic: false
},
self
));
});
};
SchemaAlignmentDialog.UINode.prototype._showColumnPopupMenu = function(elmt) {
self = this;
var menu = [];
if (!this._options.mustBeCellTopic) {
menu.push({
label: "Anonymous Node",
click: function() {
self._node.nodeType = "anonymous";
self._showExpandable();
self._renderMain();
}
});
menu.push({
label: "Freebase Topic",
click: function() {
self._node.nodeType = "topic";
self._hideExpandable();
self._renderMain();
}
});
menu.push({
label: "Value",
click: function() {
self._node.nodeType = "value";
self._hideExpandable();
self._renderMain();
}
});
menu.push({}); // separator
}
var columns = theProject.columnModel.columns;
var createColumnMenuItem = function(index) {
menu.push({
label: columns[index].name,
click: function() {
self._node.nodeType = "cell-as-topic";
self._node.columnNames = [ columns[index].name ];
self._showExpandable();
self._renderMain();
}
});
};
for (var i = 0; i < columns.length; i++) {
createColumnMenuItem(i);
}
MenuSystem.createAndShowStandardMenu(menu, elmt);
};
SchemaAlignmentDialog.UINode.prototype._showNodeConfigDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("800px");
var header = $('<div></div>').addClass("dialog-header").text("Schema Skeleton Node").appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
/*--------------------------------------------------
* Body
*--------------------------------------------------
*/
var literalTypeSelectHtml =
'<option value="/type/text" checked>text</option>' +
'<option value="/type/int">int</option>' +
'<option value="/type/float">float</option>' +
'<option value="/type/double">double</option>' +
'<option value="/type/boolean">boolean</option>' +
'<option value="/type/datetime">date/time</option>';
var html = $(
'<table class="grid-layout layout-normal layout-full">' +
'<tr>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td>' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="cell-as" bind="radioNodeTypeCellAs" /> Set to Cell in Column' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td><div class="schema-alignment-node-dialog-column-list" bind="divColumns"></div></td>' +
'<td>' +
'<table class="grid-layout layout-tight" cols="4">' +
'<tr>' +
'<td colspan="4">The cell\'s content is used ...</td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-topic" bind="radioNodeTypeCellAsTopic" /></td>' +
'<td colspan="3">to specify a Freebase topic, as reconciled</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="1" width="1%"><input type="checkbox" bind="radioNodeTypeCellAsTopicCreate" /></td>' +
'<td colspan="2">If not reconciled, create new topic named by the cell\'s content, and assign it a type</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td></td>' +
'<td colspan="1">Type:</td>' +
'<td colspan="1"><input bind="cellAsTopicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-value" bind="radioNodeTypeCellAsValue" /></td>' +
'<td colspan="3">as a literal value</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Literal type</td>' +
'<td colspan="1"><select bind="cellAsValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Language (for text)</td>' +
'<td colspan="1"><input bind="cellAsValueLanguageInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-key" bind="radioNodeTypeCellAsKey" /></td>' +
'<td colspan="3">as a key in a namespace</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Namespace</td>' +
'<td colspan="1"><input bind="cellAsKeyInput" /></td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>' +
'</td>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="anonymous" bind="radioNodeTypeAnonymous" /> Generate an anonymous graph node' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Assign a type to the node</td>' +
'<td>&nbsp;<input bind="anonymousNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="topic" bind="radioNodeTypeTopic" /> Use one existing Freebase topic' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Topic</td>' +
'<td><input bind="topicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="value" bind="radioNodeTypeValue" /> Use a literal value' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Value</td>' +
'<td><input bind="valueNodeTypeValueInput" /></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Value type</td>' +
'<td><select bind="valueNodeTypeValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Language</td>' +
'<td><input bind="valueNodeTypeLanguageInput" /></td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>'
).appendTo(body);
var elmts = DOM.bind(html);
var tableColumns = $('<table></table>')
.attr("cellspacing", "5")
.attr("cellpadding", "0")
.appendTo(elmts.divColumns)[0];
var columnMap = {};
if ("columnNames" in self._node) {
for (var i = 0; i < self._node.columnNames.length; i++) {
columnMap[self._node.columnNames[i]] = true;
}
}
var makeColumnChoice = function(column, columnIndex) {
var tr = tableColumns.insertRow(tableColumns.rows.length);
var radio = $('<input />')
.attr("type", "checkbox")
.attr("value", column.name)
.attr("name", "schema-align-node-dialog-column")
.appendTo(tr.insertCell(0))
.click(function() {
elmts.radioNodeTypeCellAs[0].checked = true;
if ("reconConfig" in column) {
var typeID = column.reconConfig.type.id;
var typeName = column.reconConfig.type.name;
elmts.cellAsTopicNodeTypeInput[0].value = typeName;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", { "id" : typeID, "name" : typeName });
elmts.radioNodeTypeCellAsTopicCreate[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
}
});
if (column.name in columnMap) {
radio.attr("checked", "true");
}
$('<span></span>').text(column.name).appendTo(tr.insertCell(1));
};
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
makeColumnChoice(columns[i], i);
}
elmts.anonymousNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeAnonymous[0].checked = true; })
.suggestT({ type: "/type/type" });
elmts.topicNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeTopic[0].checked = true; })
.suggest({});
elmts.valueNodeTypeValueInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeValueTypeSelect
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeLanguageInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; })
.suggest({ type: "/type/lang" });
elmts.radioNodeTypeCellAsTopicCreate
.click(function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
});
elmts.cellAsTopicNodeTypeInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
})
.suggestT({ type: "/type/type" });
elmts.cellAsValueTypeSelect
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
});
elmts.cellAsValueLanguageInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
})
.suggest({ type: "/type/lang" });
elmts.cellAsKeyInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsKey[0].checked = true;
})
.suggest({ type: "/type/namespace" });
elmts.radioNodeTypeCellAsTopic[0].checked = true; // just make sure some subtype is selected
if (this._node.nodeType.match(/^cell-as-/)) {
elmts.radioNodeTypeCellAs[0].checked = true;
if (this._node.nodeType == "cell-as-topic") {
elmts.radioNodeTypeCellAsTopic[0].checked = true;
} else if (this._node.nodeType == "cell-as-value") {
elmts.radioNodeTypeCellAsValue[0].checked = true;
} else if (this._node.nodeType == "cell-as-key") {
elmts.radioNodeTypeCellAsKey[0].checked = true;
}
} else if (this._node.nodeType == "anonymous") {
elmts.radioNodeTypeAnonymous[0].checked = true;
} else if (this._node.nodeType == "topic") {
elmts.radioNodeTypeTopic[0].checked = true;
} else if (this._node.nodeType == "value") {
elmts.radioNodeTypeValue[0].checked = true;
}
if ("type" in this._node) {
elmts.anonymousNodeTypeInput[0].value = this._node.type.name;
elmts.anonymousNodeTypeInput.data("data.suggest", this._node.type);
elmts.cellAsTopicNodeTypeInput[0].value = this._node.type.name;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", this._node.type);
}
if ("topic" in this._node) {
elmts.topicNodeTypeInput[0].value = this._node.topic.name;
elmts.topicNodeTypeInput.data("data.suggest", this._node.topic);
}
if ("namespace" in this._node) {
elmts.cellAsKeyInput[0].value = this._node.namespace.name;
elmts.cellAsKeyInput.data("data.suggest", this._node.namespace);
}
if ("createForNoReconMatch" in this._node) {
elmts.radioNodeTypeCellAsTopicCreate[0].checked = this._node.createForNoReconMatch;
}
if ("lang" in this._node) {
elmts.valueNodeTypeLanguageInput[0].value = this._node.lang;
elmts.valueNodeTypeLanguageInput.data("data.suggest", { id: this._node.lang });
elmts.cellAsValueLanguageInput[0].value = this._node.lang;
elmts.cellAsValueLanguageInput.data("data.suggest", { id: this._node.lang });
}
if ("valueType" in this._node) {
elmts.valueNodeTypeValueTypeSelect[0].value = this._node.valueType;
elmts.cellAsValueTypeSelect[0].value = this._node.valueType;
}
/*--------------------------------------------------
* Footer
*--------------------------------------------------
*/
var getResultJSON = function() {
var node = {
nodeType: $("input[name='schema-align-node-dialog-node-type']:checked")[0].value
};
if (node.nodeType == "cell-as") {
node.nodeType = $("input[name='schema-align-node-dialog-node-subtype']:checked")[0].value;
node.columnNames = $("input[name='schema-align-node-dialog-column']:checked").map(function() {
return this.getAttribute("value");
}).get();
if (node.columnNames.length == 0) {
alert("You must select at least one column.");
return null;
}
if (node.nodeType == "cell-as-topic") {
node.createForNoReconMatch = elmts.radioNodeTypeCellAsTopicCreate[0].checked;
var t = elmts.cellAsTopicNodeTypeInput.data("data.suggest");
if (!(t) && node.createForNoReconMatch) {
alert("For creating a new graph node, you need to specify a type for it.");
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "cell-as-value") {
node.valueType = elmts.cellAsValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.cellAsValueLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
} else if (node.nodeType == "cell-as-key") {
var t = elmts.cellAsKeyInput.data("data.suggest");
if (!(t)) {
alert("Please specify the namespace.");
return null;
}
node.namespace = {
id: t.id,
name: t.name
};
}
} else if (node.nodeType == "anonymous") {
var t = elmts.anonymousNodeTypeInput.data("data.suggest");
if (!(t)) {
alert("For generating an anonymous graph node, you need to specify a type for it.");
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "topic") {
var t = elmts.topicNodeTypeInput.data("data.suggest");
if (!(t)) {
alert("Please specify which existing Freebase topic to use.");
return null;
}
node.topic = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "value") {
node.value = $.trim(elmts.valueNodeTypeValueInput[0].value);
if (!node.value.length) {
alert("Please specify the value to use.");
return null;
}
node.valueType = elmts.valueNodeTypeValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.valueNodeTypeLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
}
return node;
};
$('<button></button>').html("&nbsp;&nbsp;OK&nbsp;&nbsp;").click(function() {
var node = getResultJSON();
if (node !== null) {
DialogSystem.dismissUntil(level - 1);
self._node = node;
self.render();
self._dialog.preview();
}
}).appendTo(footer);
$('<button></button>').text("Cancel").click(function() {
DialogSystem.dismissUntil(level - 1);
}).appendTo(footer);
var level = DialogSystem.showDialog(frame);
};
SchemaAlignmentDialog.UINode.prototype.getJSON = function() {
var result = null;
var getLinks = false;
if (this._node.nodeType.match(/^cell-as-/)) {
if (!("columnNames" in this._node) || !this._node.columnNames) {
return null;
}
if (this._node.nodeType == "cell-as-topic") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
type: "type" in this._node ? cloneDeep(this._node.type) : { "id" : "/common/topic", "name" : "Topic", "cvt" : false },
createForNoReconMatch: "createForNoReconMatch" in this._node ? this._node.createForNoReconMatch : true
};
getLinks = true;
} else if (this._node.nodeType == "cell-as-value") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "cell-as-key") {
if (!("namespace" in this._node) || !this._node.namespace) {
return null;
}
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
namespace: cloneDeep(this._node.namespace)
};
}
} else if (this._node.nodeType == "topic") {
if (!("topic" in this._node) || !this._node.topic) {
return null;
}
result = {
nodeType: this._node.nodeType,
topic: cloneDeep(this._node.topic)
};
getLinks = true;
} else if (this._node.nodeType == "value") {
if (!("value" in this._node) || !this._node.value) {
return null;
}
result = {
nodeType: this._node.nodeType,
value: this._node.value,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "anonymous") {
if (!("type" in this._node) || !this._node.type) {
return null;
}
result = {
nodeType: this._node.nodeType,
type: cloneDeep(this._node.type)
};
getLinks = true;
}
if (!result) {
return null;
}
if (getLinks) {
var links = [];
for (var i = 0; i < this._linkUIs.length; i++) {
var link = this._linkUIs[i].getJSON();
if (link !== null) {
links.push(link);
}
}
result.links = links;
}
return result;
};

289
main/webapp/modules/core/scripts/protograph/schema-alignment.js
View File

@ -1,289 +0,0 @@
var SchemaAlignment = {};
SchemaAlignment.autoAlign = function() {
var protograph = {};
var columns = theProject.columnModel.columns;
var typedCandidates = [];
var candidates = [];
for (var c = 0; c < columns.length; c++) {
var column = columns[c];
var typed = (column.reconConfig) &&
ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace);
var candidate = {
status: "unbound",
typed: typed,
index: c,
column: column
};
candidates.push(candidate);
if (typed) {
typedCandidates.push(candidate);
}
}
if (typedCandidates.length > 0) {
} else {
var queries = {};
for (var i = 0; i < candidates.length; i++) {
var candidate = candidates[i];
var name = SchemaAlignment._cleanName(candidate.column.name);
var key = "t" + i + ":search";
queries[key] = {
"query" : name,
"limit" : 10,
"type" : "/type/type,/type/property",
"type_strict" : "any"
};
}
SchemaAlignment._batchSearch(queries, function(result) {
console.log(result);
});
}
};
SchemaAlignment._batchSearch = function(queries, onDone) {
var keys = [];
for (var n in queries) {
if (queries.hasOwnProperty(n)) {
keys.push(n);
}
}
var result = {};
var args = [];
var makeBatch = function(keyBatch) {
var batch = {};
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
batch[key] = queries[key];
}
args.push("http://api.freebase.com/api/service/search?" +
$.param({ "queries" : JSON.stringify(batch) }) + "&callback=?");
args.push(null); // no data
args.push(function(data) {
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
result[key] = data[key];
}
});
};
for (var i = 0; i < keys.length; i += 10) {
makeBatch(keys.slice(i, i + 10));
}
args.push(function() {
onDone(result);
});
Ajax.chainGetJSON.apply(null, args);
};
SchemaAlignment._cleanName = function(s) {
return s.replace(/\W/g, " ").replace(/\s+/g, " ").toLowerCase();
};
SchemaAlignment.createNewRootNode = function() {
var rootNode = null;
var links = [];
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
var target = {
nodeType: "cell-as-topic",
columnName: column.name,
createForNoReconMatch: true
};
if ((column.reconConfig) &&
ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace) &&
(column.reconConfig.type)) {
target.type = {
id: column.reconConfig.type.id,
name: column.reconConfig.type.name
};
}
if (column.name == theProject.columnModel.keyColumnName) {
rootNode = target;
} else {
links.push({
property: null,
target: target
});
}
}
rootNode = rootNode || { nodeType: "cell-as-topic" };
rootNode.links = links;
return rootNode;
};
function SchemaAlignmentDialog(protograph, onDone) {
this._onDone = onDone;
this._originalProtograph = protograph || { rootNodes: [] };
this._protograph = cloneDeep(this._originalProtograph); // this is what can be munched on
if (!this._protograph.rootNodes.length) {
this._protograph.rootNodes.push(SchemaAlignment.createNewRootNode());
}
this._nodeUIs = [];
this._createDialog();
this.preview();
}
SchemaAlignmentDialog.prototype._createDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("1000px");
var header = $('<div></div>').addClass("dialog-header").text("Schema Alignment").appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
this._constructFooter(footer);
this._constructBody(body);
this._level = DialogSystem.showDialog(frame);
this._renderBody(body);
};
SchemaAlignmentDialog.prototype._constructFooter = function(footer) {
var self = this;
$('<button></button>').html("&nbsp;&nbsp;OK&nbsp;&nbsp;").click(function() {
var protograph = self.getJSON();
Refine.postCoreProcess(
"save-protograph",
{},
{ protograph: JSON.stringify(protograph) },
{},
{
onDone: function() {
DialogSystem.dismissUntil(self._level - 1);
theProject.overlayModels.freebaseProtograph = protograph;
}
}
);
}).appendTo(footer);
$('<button></button>').text("Cancel").click(function() {
DialogSystem.dismissUntil(self._level - 1);
}).appendTo(footer);
};
SchemaAlignmentDialog.prototype._constructBody = function(body) {
$('<p>' +
'The schema alignment skeleton below specifies how the graph-shaped data that will get generated ' +
'from your grid-shaped data and written into Freebase. The cells in each record of your data will ' +
'get placed into nodes within the skeleton. Configure the skeleton by specifying which ' +
'column to substitute into which node. A node can also be an automatically generated ' +
'anonymous node, or it can be an explicit value or topic that is the same for all records.' +
'</p>').appendTo(body);
$(
'<div id="schema-alignment-tabs" class="refine-tabs">' +
'<ul>' +
'<li><a href="#schema-alignment-tabs-protograph">Skeleton</a></li>' +
'<li><a href="#schema-alignment-tabs-preview-mqllike">MQL-like Preview</a></li>' +
'<li><a href="#schema-alignment-tabs-preview-tripleloader">TripleLoader Preview</a></li>' +
'</ul>' +
'<div id="schema-alignment-tabs-protograph">' +
'<div class="schema-alignment-dialog-canvas"></div>' +
'</div>' +
'<div id="schema-alignment-tabs-preview-mqllike" style="display: none;">' +
'<div class="schema-alignment-dialog-preview"></div>' +
'</div>' +
'<div id="schema-alignment-tabs-preview-tripleloader" style="display: none;">' +
'<div class="schema-alignment-dialog-preview"></div>' +
'</div>' +
'</div>'
).appendTo(body);
};
SchemaAlignmentDialog.prototype._renderBody = function(body) {
var self = this;
$("#schema-alignment-tabs").tabs();
$("#schema-alignment-tabs-preview-mqllike").css("display", "");
$("#schema-alignment-tabs-preview-tripleloader").css("display", "");
this._canvas = $(".schema-alignment-dialog-canvas");
this._nodeTable = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._canvas)[0];
for (var i = 0; i < this._protograph.rootNodes.length; i++) {
this._nodeUIs.push(new SchemaAlignmentDialog.UINode(
this,
this._protograph.rootNodes[i],
this._nodeTable,
{
expanded: true,
mustBeCellTopic: true
}
));
}
this._previewPanes = $(".schema-alignment-dialog-preview");
};
SchemaAlignmentDialog.prototype.getJSON = function() {
var rootNodes = [];
for (var i = 0; i < this._nodeUIs.length; i++) {
var node = this._nodeUIs[i].getJSON();
if (node !== null) {
rootNodes.push(node);
}
}
return {
rootNodes: rootNodes
};
};
SchemaAlignmentDialog.prototype.preview = function() {
var self = this;
this._previewPanes.empty();
var protograph = this.getJSON();
$.post(
"/command/core/preview-protograph?" + $.param({ project: theProject.id }),
{ protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
if ("mqllike" in data) {
$(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2));
}
if ("tripleloader" in data) {
$(self._previewPanes[1]).text(data.tripleloader);
}
},
"json"
);
};
SchemaAlignmentDialog._findColumn = function(cellIndex) {
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
if (column.cellIndex == cellIndex) {
return column;
}
}
return null;
};

30
main/webapp/modules/core/scripts/util/freebase.js
View File

@ -1,30 +0,0 @@
var Freebase = {};
Freebase.mqlread = function(query, options, onDone) {
var params = {};
var queryEnv = {
"query": query
};
if (options) {
for (var n in options) {
if (options.hasOwnProperty(n)) {
var v = options[n];
if (typeof v != "string") {
v = JSON.stringify(v);
}
queryEnv[n] = v;
}
}
}
params.query = JSON.stringify(queryEnv);
$.getJSON(
"http://api.freebase.com/api/service/mqlread?" + $.param(params) + "&callback=?",
null,
onDone,
"jsonp"
);
};

120
main/webapp/modules/core/scripts/util/sign.js
View File

@ -1,120 +0,0 @@
if (typeof window.Sign == 'undefined') {
window.Sign = {
window_position: function() {
var position = {};
if (typeof(window.innerWidth) == 'number') {
// Non-IE
position.width = window.outerWidth;
position.height = window.outerHeight;
position.top = window.screenY;
position.left = window.screenX;
} else if (document.documentElement && (document.documentElement.clientWidth || document.documentElement.clientHeight)) {
// IE 6+ in 'standards compliant mode'
position.width = document.body.clientWidth;
position.height = document.body.clientHeight;
position.top = window.screenTop;
position.left = window.screenLeft;
}
return position;
},
popup : function(url, width, height, windowname) {
width = width || 700;
height = height || 500;
var pos = window.Sign.window_position();
var left = Math.floor((pos.width - width) / 2) + pos.left;
var top = Math.floor((pos.height - height) / 2) + pos.top;
// Chrome might fix this bug, but until then add some padding
// to the height of the popup for the urlbar
var is_chrome = /chrome/.test(navigator.userAgent.toLowerCase());
if (is_chrome) {
height += 50;
}
var params = {
width: width,
height: height,
top: top,
left: left,
directories: 'no',
location: 'no',
menubar: 'no',
resizable: 'no',
scrollbars: 'yes',
status: 'no',
toolbar: 'no'
};
var params_list = [];
for (var key in params) {
if (params.hasOwnProperty(key)) {
params_list.push(key + "=" + params[key]);
}
}
return window.open(url, windowname || "", params_list.join(","));
},
signintize : function(cont) {
$('.signedin').show();
$('.signedout').hide();
if (window.user) {
$('.user').html('<a href="http://freebase.com/view' + window.user.id + '">' + window.user.username + '</a>');
}
if (typeof cont == 'function') cont();
},
signin : function(success, provider, width, height) {
var newwin = window.Sign.popup("/command/core/authorize/" + provider, width, height);
if (newwin !== null) {
newwin.opener = window;
}
window.onauthorization = function() {
if (typeof success == 'undefined') {
window.location.reload();
} else {
$.ajax({
url: "/command/core/check-authorization/" + provider,
dataType: "json",
success: function(data) {
window.user = data;
window.Sign.signintize(success);
}
});
}
};
if (window.focus && newwin !== null) {
newwin.focus();
}
return false;
},
signoutize : function(cont) {
$('.signedin').hide();
$('.signedout').show();
if (typeof cont == 'function') cont();
},
signout : function(success,provider) {
$.ajax({
url: "/command/core/deauthorize/" + provider,
success: function() {
if (typeof success == 'undefined') {
window.location.reload();
} else {
window.Sign.signoutize(success);
}
}
});
}
};
}

13
main/webapp/modules/core/scripts/views/data-table/menu-edit-cells.js
View File

@ -119,27 +119,33 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
label: "Common Transforms",
submenu: [
{
id: "core/unescape-html-entities",
label: "Unescape HTML entities",
click: function() { doTextTransform("value.unescape('html')", "store-blank", true, 10); }
},
{
id: "core/collapse-whitespace",
label: "Collapse whitespace",
click: function() { doTextTransform("value.replace(/\\s+/,' ')", "store-blank", false, ""); }
},
{},
{
id: "core/to-titlecase",
label: "To Titlecase",
click: function() { doTextTransform("toTitlecase(value)", "store-blank", false, ""); }
},
{
id: "core/to-uppercase",
label: "To Uppercase",
click: function() { doTextTransform("toUppercase(value)", "store-blank", false, ""); }
},
{
id: "core/to-lowercase",
label: "To Lowercase",
click: function() { doTextTransform("toLowercase(value)", "store-blank", false, ""); }
},
{
id: "core/to-blank",
label: "To Blank",
click: function() { doTextTransform("null", "store-blank", false, ""); }
}
@ -147,24 +153,29 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
},
{},
{
id: "core/fill-down",
label: "Fill Down",
click: doFillDown
},
{
id: "core/blank-down",
label: "Blank Down",
click: doBlankDown
},
{},
{
id: "core/split-multi-valued-cells",
label: "Split Multi-Valued Cells ...",
click: doSplitMultiValueCells
},
{
id: "core/join-multi-valued-cells",
label: "Join Multi-Valued Cells ...",
click: doJoinMultiValueCells
},
{},
{
id: "core/cluster",
label: "Cluster & Edit ...",
click: function() { new ClusteringDialog(column.name, "value"); }
}
@ -266,10 +277,12 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
MenuSystem.appendTo(menu, [ "core/transpose" ], [
{
id: "core/transpose-columns-into-rows",
label: "Cells Across Columns into Rows",
click: doTransposeColumnsIntoRows
},
{
id: "core/transpose-rows-into-columns",
label: "Cells in Rows into Columns",
click: doTransposeRowsIntoColumns
}

35
main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js
View File

@ -89,28 +89,6 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
);
};
var doAddColumnFromFreebase = function() {
var o = DataTableView.sampleVisibleRows(column);
new ExtendDataPreviewDialog(
column,
columnIndex,
o.rowIndices,
function(extension) {
Refine.postCoreProcess(
"extend-data",
{
baseColumnName: column.name,
columnInsertIndex: columnIndex + 1
},
{
extension: JSON.stringify(extension)
},
{ rowsChanged: true, modelsChanged: true }
);
}
);
};
var doRemoveColumn = function() {
Refine.postCoreProcess(
"remove-column",
@ -226,45 +204,50 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
MenuSystem.appendTo(menu, [ "core/edit-column" ], [
{
id: "core/split-column",
label: "Split into Several Columns ...",
click: doSplitColumn
},
{},
{
id: "core/add-column",
label: "Add Column Based on This Column ...",
click: doAddColumn
},
{
label: "Add Columns From Freebase ...",
click: doAddColumnFromFreebase
},
{
id: "core/add-column-by-fetching-urls",
label: "Add Column By Fetching URLs ...",
click: doAddColumnByFetchingURLs
},
{},
{
id: "core/rename-column",
label: "Rename This Column",
click: doRenameColumn
},
{
id: "core/remove-column",
label: "Remove This Column",
click: doRemoveColumn
},
{},
{
id: "core/move-column-to-beginning",
label: "Move Column to Beginning",
click: function() { doMoveColumnTo(0); }
},
{
id: "core/move-column-to-end",
label: "Move Column to End",
click: function() { doMoveColumnTo(theProject.columnModel.columns.length - 1); }
},
{
id: "core/move-column-to-left",
label: "Move Column Left",
click: function() { doMoveColumnBy(-1); }
},
{
id: "core/move-column-to-right",
label: "Move Column Right",
click: function() { doMoveColumnBy(1); }
}

21
main/webapp/modules/core/scripts/views/data-table/menu-reconcile.js
View File

@ -84,15 +84,18 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
MenuSystem.appendTo(menu, [ "core/reconcile" ], [
{
id: "core/reconcile",
label: "Start Reconciling ...",
tooltip: "Reconcile text in this column with topics on Freebase",
click: doReconcile
},
{},
{
id: "core/facets",
label: "Facets",
submenu: [
{
id: "core/by-judgment",
label: "By Judgment",
click: function() {
ui.browsingEngine.addFacet(
@ -111,7 +114,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
},
{},
{
label: "Best Candidate's Relevance Score",
id: "core/by-best-candidates-score",
label: "Best Candidate's Score",
click: function() {
ui.browsingEngine.addFacet(
"range",
@ -127,6 +131,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/by-best-candidates-type-match",
label: "Best Candidate's Type Match",
click: function() {
ui.browsingEngine.addFacet(
@ -144,6 +149,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/by-best-candidates-name-match",
label: "Best Candidate's Name Match",
click: function() {
ui.browsingEngine.addFacet(
@ -162,6 +168,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
},
{},
{
id: "core/by-best-candidates-name-edit-distance",
label: "Best Candidate's Name Edit Distance",
click: function() {
ui.browsingEngine.addFacet(
@ -178,6 +185,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/by-best-candidates-name-word-similarity",
label: "Best Candidate's Name Word Similarity",
click: function() {
ui.browsingEngine.addFacet(
@ -195,6 +203,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
},
{},
{
id: "core/by-best-candidates-types",
label: "Best Candidate's Types",
click: function() {
ui.browsingEngine.addFacet(
@ -211,9 +220,11 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
]
},
{
id: "core/qa-facets",
label: "QA Facets",
submenu: [
{
id: "core/by-qa-results",
label: "QA Results",
click: function() {
ui.browsingEngine.addFacet(
@ -227,6 +238,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/by-judgment-actions",
label: "Judgment Actions",
click: function() {
ui.browsingEngine.addFacet(
@ -240,6 +252,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/by-judgment-history-entries",
label: "Judgment History Entries",
click: function() {
ui.browsingEngine.addFacet(
@ -255,14 +268,17 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
]
},
{
id: "core/actions",
label: "Actions",
submenu: [
{
id: "core/match-to-best-candidate",
label: "Match Each Cell to Its Best Candidate",
tooltip: "Match each cell to its best candidate in this column for all current filtered rows",
click: doReconMatchBestCandidates
},
{
id: "core/match-to-new-topic",
label: "Create a New Topic for Each Cell",
tooltip: "Mark to create one new topic for each cell in this column for all current filtered rows",
click: function() {
@ -271,6 +287,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
},
{},
{
id: "core/match-similar-to-new-topic",
label: "Create One New Topic for Similar Cells",
tooltip: "Mark to create one new topic for each group of similar cells in this column for all current filtered rows",
click: function() {
@ -278,12 +295,14 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
}
},
{
id: "core/match-to-specific",
label: "Match All Filtered Cells to ...",
tooltip: "Search for a topic to match all filtered cells to",
click: doSearchToMatch
},
{},
{
id: "core/discard-judgments",
label: "Discard Reconciliation Judgments",
tooltip: "Discard reconciliaton judgments in this column for all current filtered rows",
click: doReconDiscardJudgments

38
main/webapp/modules/core/styles/dialogs/extend-data-preview-dialog.less
View File

@ -1,38 +0,0 @@
@import-less url("../theme.less");
.extend-data-preview-dialog .suggested-property-container {
border: 1px solid #aaa;
padding: 5px;
overflow: auto;
height: 400px;
}
.extend-data-preview-dialog .suggested-property {
padding: 5px;
}
.extend-data-preview-dialog input.property-suggest {
display: block;
padding: 2%;
width: 96%;
}
.extend-data-preview-dialog .preview-container {
border: 1px solid #aaa;
overflow: auto;
}
.extend-data-preview-dialog .preview-container table {
border-collapse: collapse;
}
.extend-data-preview-dialog .preview-container td, .extend-data-preview-dialog .preview-container th {
padding: 3px 5px;
border-bottom: 1px solid #ddd;
border-right: 1px solid #ddd;
}
.extend-data-preview-dialog .preview-container th img {
vertical-align: top;
margin-left: 5px;
}

60
main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.less
View File

@ -1,60 +0,0 @@
@import-less url("../theme.less");
.freebase-loading-tripleloader-data {
height: 400px;
width: 99%;
overflow: auto;
border: 1px solid #aaa;
white-space: pre;
padding: 0.3em 0.5em 0.5em 0.5em;
font-family: monospace;
}
.freebase-loading-tripleloader-message {
height: 500px;
overflow: auto;
border: 1px solid #aaa;
padding: 0.3em 0.5em 0.5em 0.5em;
}
.freebase-loading-tripleloader-message h2 {
text-align: center;
font-size: 150%;
margin-top: 3em;
font-weight: normal;
}
.freebase-loading-tripleloader-message h2 span {
font-size; 130%;
font-weight: bold;
}
.freebase-loading-tripleloader-message h4 {
text-align: center;
font-size: 120%;
margin-top: 2em;
font-weight: normal;
}
.freebase-loading-tripleloader-message a {
font-size: 120%;
font-weight: bold;
}
.freebase-loading-authorization {
margin: 0em 2em;
padding: 0.6em 0.8em;
border: 1px solid #ccc;
background-color: #fff;
-moz-border-radius: 0.8em;
-webkit-border-radius: 0.8em;
}
.freebase-loading-tripleloader-info {
margin-bottom: 0.5em;
}
.freebase-loading-tripleloader-info textarea {
width: 99%;
height: 5em;
}

114
main/webapp/modules/core/styles/protograph/schema-alignment-dialog.less
View File

@ -1,114 +0,0 @@
@import-less url("../theme.less");
.schema-alignment-dialog-canvas {
height: 400px;
overflow: auto;
border: 1px solid #aaa;
padding: 10px;
background: white;
}
table.schema-alignment-table-layout {
border-collapse: collapse;
margin: 0px;
padding: 0px;
margin-bottom: 1em;
}
table.schema-alignment-table-layout .padded {
padding: 3px 6px;
}
div.schema-alignment-detail-container {
border-left: 3px solid #eee;
}
td.schema-alignment-node-main, td.schema-alignment-link-main {
white-space: pre;
width: 300px;
}
td.schema-alignment-node-toggle, td.schema-alignment-link-toggle {
white-space: pre;
width: 1%;
}
td.schema-alignment-node-toggle img, td.schema-alignment-link-toggle img {
cursor: pointer;
vertical-align: middle;
padding: 2px;
}
td.schema-alignment-node-details, td.schema-alignment-link-details {
width: 90%;
}
a.schema-alignment-node-tag {
padding: 3px 6px;
background: #ddd;
text-decoration: none;
color: black;
-moz-border-radius: 10px;
}
a.schema-alignment-node-tag:hover {
background: #888;
color: white;
}
.schema-alignment-node-column {
font-weight: bold;
}
a.schema-alignment-link-tag {
padding: 3px 6px;
text-decoration: none;
color: black;
}
a.schema-alignment-link-tag:hover {
color: #88f;
}
div.schema-alignment-dialog-preview {
height: 400px;
overflow: auto;
border: 1px solid #aaa;
background: white;
padding: 10px;
white-space: pre;
font-family: monospace;
font-size: 9pt;
}
/*--------------------------------------------------
* Node dialog
*--------------------------------------------------
*/
.schema-align-node-dialog-node-type {
padding: 0.25em;
background: #ddf;
-moz-border-radius: 0.5em;
}
.schema-align-node-dialog-node-type input {
vertical-align: text-bottom;
}
.schema-alignment-node-dialog-column-list {
height: 300px;
width: 200px;
overflow: auto;
border: 1px solid #ddd;
}
/*--------------------------------------------------
* Link dialog
*--------------------------------------------------
*/
.schema-alignment-link-menu-section {
padding: 8px;
border-bottom: 1px solid #ddd;
margin-bottom: 3px;
}
.schema-alignment-link-menu-section-last {
padding: 8px;
}