s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
Tom Morris 2015-09-25 17:14:48 -04:00
commit 8511c6c30d
113 changed files with 441 additions and 11324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
.classpath
View File

@ -1,90 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="main/src"/>
<classpathentry kind="src" path="server/src"/>
<classpathentry kind="src" path="extensions/freebase/src"/>
<classpathentry kind="src" path="extensions/gdata/src"/>
<classpathentry kind="src" path="extensions/jython/src"/>
<classpathentry kind="src" path="extensions/pc-axis/src"/>
<classpathentry kind="src" path="extensions/sample/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/ant-tools-1.8.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/arithcode-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/clojure-1.5.1-slim.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-codec-1.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/lessen-trunk-r8.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/log4j-1.2.15.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/marc4j-2.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/opencsv-2.4-SNAPSHOT.jar" sourcepath="main/webapp/WEB-INF/lib-src/opencsv-2.4-SNAPSHOT-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/poi-3.8-20120326.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.8-20120326.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.8-20120326.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/rhino-1.7R2.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/secondstring-20100303.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/slf4j-api-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/velocity-1.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/vicino-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xmlbeans-2.3.0.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jdatapath-alpha2.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jetty-6.1.22.jar" sourcepath="server/lib-src/jetty-6.1.22-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jetty-util-6.1.22.jar" sourcepath="server/lib-src/jetty-util-6.1.22-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/log4j-1.2.15.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/servlet-api-2.5.jar" sourcepath="server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-api-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.5.3.jar"/>
<classpathentry exported="true" kind="lib" path="broker/core/module/MOD-INF/lib/bdb-je-4.0.103.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jsoup-1.4.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/tests/server/lib/mockito-all-1.9.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/tests/server/lib/testng-6.8.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/mail.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/odfdom-java-0.8.7.jar" sourcepath="main/webapp/WEB-INF/lib-src/odfdom-java-0.8.7-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/resolver.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/serializer.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xercesImpl-2.11.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xml-apis.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpclient-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpcore-4.2.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-core-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/guava-13.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar" sourcepath="extensions/freebase/module/MOD-INF/libsrc/google-api-services-freebase-v1-rev25-1.13.2-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jsr305-1.3.9.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.2.5.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar" sourcepath="C:/Users/tfmorris/.sourceattacher/google-api-services-drive-v2-rev168-1.20.0-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jackson-core-2.1.3.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
<classpathentry kind="output" path="build"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="main/src"/>
<classpathentry kind="src" path="server/src"/>
<classpathentry kind="src" path="extensions/gdata/src"/>
<classpathentry kind="src" path="extensions/jython/src"/>
<classpathentry kind="src" path="extensions/pc-axis/src"/>
<classpathentry kind="src" path="extensions/sample/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/ant-tools-1.8.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/arithcode-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/clojure-1.5.1-slim.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-codec-1.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/lessen-trunk-r8.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/log4j-1.2.15.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/marc4j-2.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/opencsv-2.4-SNAPSHOT.jar" sourcepath="main/webapp/WEB-INF/lib-src/opencsv-2.4-SNAPSHOT-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/rhino-1.7R2.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/secondstring-20100303.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/slf4j-api-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/velocity-1.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/vicino-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xmlbeans-2.3.0.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jdatapath-alpha2.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jetty-6.1.22.jar" sourcepath="server/lib-src/jetty-6.1.22-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/jetty-util-6.1.22.jar" sourcepath="server/lib-src/jetty-util-6.1.22-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/log4j-1.2.15.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/servlet-api-2.5.jar" sourcepath="server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-api-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.5.3.jar"/>
<classpathentry exported="true" kind="lib" path="broker/core/module/MOD-INF/lib/bdb-je-4.0.103.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jsoup-1.4.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/tests/server/lib/mockito-all-1.9.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/tests/server/lib/testng-6.8.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/mail.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/odfdom-java-0.8.7.jar" sourcepath="main/webapp/WEB-INF/lib-src/odfdom-java-0.8.7-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/resolver.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/serializer.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xercesImpl-2.11.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xml-apis.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpclient-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpcore-4.2.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-core-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/guava-13.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jsr305-1.3.9.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.2.5.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar" sourcepath="C:/Users/tfmorris/.sourceattacher/google-api-services-drive-v2-rev168-1.20.0-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jackson-core-2.1.3.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.12-20150511.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.12-20150511.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/EditDistanceJoiner.jar"/>
<classpathentry kind="output" path="build"/>
</classpath>

2
IDEs/eclipse/Refine with extensions.launch
View File

File diff suppressed because one or more lines are too long

4
README.md
View File

@ -3,8 +3,8 @@
[![Build Status](https://travis-ci.org/OpenRefine/OpenRefine.png?branch=master)](https://travis-ci.org/OpenRefine/OpenRefine) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=32795)](https://www.bountysource.com/trackers/32795-open-refine?utm_source=32795&utm_medium=shield&utm_campaign=TRACKER_BADGE)
OpenRefine is a power tool that allows you to load data, understand it,
clean it up, reconcile it to master database, and augment it with data coming from
Freebase or other web sources. All with the comfort and privacy of
clean it up, reconcile it, and augment it with data coming from
the web. All with the comfort and privacy of
your own computer.

2
extensions/build.xml
View File

@ -11,7 +11,6 @@
<echo message="Building extensions" />
<ant dir="sample/" target="build" />
<ant dir="jython/" target="build" />
<!-- <ant dir="freebase/" target="build" /> -->
<ant dir="gdata/" target="build" />
<ant dir="pc-axis/" target="build" />
</target>
@ -20,7 +19,6 @@
<echo message="cleaning extensions" />
<ant dir="sample/" target="clean" />
<ant dir="jython/" target="clean" />
<ant dir="freebase/" target="clean" />
<ant dir="gdata/" target="clean" />
<ant dir="pc-axis/" target="clean" />
</target>

23
extensions/freebase/.classpath
View File

@ -1,23 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry combineaccessrules="false" kind="src" path="/grefine"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry kind="lib" path="/grefine-server/lib/servlet-api-2.5.jar" sourcepath="/grefine-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/grefine-server/lib/slf4j-api-1.5.6.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/httpclient-4.2.5.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/httpclient-4.2.5-sources.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/httpcore-4.2.4.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/httpcore-4.2.4-sources.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/google-api-client-1.13.2-beta.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/google-http-client-1.13.1-beta.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/google-oauth-client-servlet-1.13.1-beta.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/mail.jar"/>
<classpathentry kind="output" path="module/MOD-INF/classes"/>
</classpath>

17
extensions/freebase/.project
View File

@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>grefine-freebase</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

12
extensions/freebase/.settings/org.eclipse.jdt.core.prefs
View File

@ -1,12 +0,0 @@
#Thu Feb 03 10:58:59 EST 2011
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.source=1.6

71
extensions/freebase/build.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--+
|
| OpenRefine Extension Freebase Build File
|
+-->
<project name="refine-freebase" default="build" basedir=".">
<property environment="env"/>
<condition property="version" value="trunk">
<not><isset property="version"/></not>
</condition>
<condition property="revision" value="rXXXX">
<not><isset property="revision"/></not>
</condition>
<condition property="full_version" value="0.0.0.0">
<not><isset property="full_version"/></not>
</condition>
<condition property="dist.dir" value="dist">
<not><isset property="dist.dir"/></not>
</condition>
<property name="fullname" value="${name}-${version}-${revision}" />
<property name="refine.dir" value="${basedir}/../../main" />
<property name="refine.webinf.dir" value="${refine.dir}/webapp/WEB-INF" />
<property name="refine.modinf.dir" value="${refine.dir}/webapp/modules/core/MOD-INF" />
<property name="refine.classes.dir" value="${refine.webinf.dir}/classes" />
<property name="refine.lib.dir" value="${refine.webinf.dir}/lib" />
<property name="server.dir" value="${basedir}/../../server" />
<property name="server.lib.dir" value="${server.dir}/lib" />
<property name="src.dir" value="${basedir}/src" />
<property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" />
<property name="classes.dir" value="${modinf.dir}/classes" />
<path id="class.path">
<fileset dir="${lib.dir}" erroronmissingdir="false">
<include name="**/*.jar" />
</fileset>
<fileset dir="${refine.lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${server.lib.dir}">
<include name="**/*.jar" />
</fileset>
<pathelement path="${refine.classes.dir}"/>
</path>
<target name="build_java">
<mkdir dir="${classes.dir}" />
<javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/>
<classpath refid="class.path" />
</javac>
</target>
<target name="build" depends="build_java"/>
<target name="clean">
<delete dir="${classes.dir}" />
</target>
</project>

1
extensions/freebase/module/MOD-INF/.gitignore vendored
View File

@ -1 +0,0 @@
/classes/

114
extensions/freebase/module/MOD-INF/controller.js
View File

@ -1,114 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
function init() {
Packages.com.google.refine.freebase.oauth.FreebaseProvider.register();
var RS = Packages.com.google.refine.RefineServlet;
RS.registerClassMapping(
"com.google.refine.model.changes.DataExtensionChange",
"com.google.refine.freebase.model.changes.DataExtensionChange");
RS.registerClassMapping(
"com.google.refine.operations.SaveProtographOperation$ProtographChange",
"com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange");
// TODO(dfhuynh): Temporary hack until we know how the core module can resolve our module's classes
RS.cacheClass(Packages.com.google.refine.freebase.model.changes.DataExtensionChange);
RS.cacheClass(Packages.com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange);
RS.registerCommand(module, "extend-data", new Packages.com.google.refine.freebase.commands.ExtendDataCommand());
RS.registerCommand(module, "preview-extend-data", new Packages.com.google.refine.freebase.commands.PreviewExtendDataCommand());
RS.registerCommand(module, "preview-protograph", new Packages.com.google.refine.freebase.commands.PreviewProtographCommand());
RS.registerCommand(module, "save-protograph", new Packages.com.google.refine.freebase.commands.SaveProtographCommand());
RS.registerCommand(module, "check-authorization", new Packages.com.google.refine.freebase.commands.auth.CheckAuthorizationCommand());
RS.registerCommand(module, "user-badges", new Packages.com.google.refine.freebase.commands.auth.GetUserBadgesCommand());
RS.registerCommand(module, "upload-data", new Packages.com.google.refine.freebase.commands.UploadDataCommand());
RS.registerCommand(module, "import-qa-data", new Packages.com.google.refine.freebase.commands.ImportQADataCommand());
RS.registerCommand(module, "mqlread", new Packages.com.google.refine.freebase.commands.MQLReadCommand());
RS.registerCommand(module, "mqlwrite", new Packages.com.google.refine.freebase.commands.MQLWriteCommand());
var OR = Packages.com.google.refine.operations.OperationRegistry;
OR.registerOperation(module, "extend-data", Packages.com.google.refine.freebase.operations.ExtendDataOperation);
OR.registerOperation(module, "import-qa-data", Packages.com.google.refine.freebase.operations.ImportQADataOperation);
OR.registerOperation(module, "save-protograph", Packages.com.google.refine.freebase.operations.SaveProtographOperation); // for backward compatibility
OR.registerOperation(module, "save-schema-alignment-skeleton", Packages.com.google.refine.freebase.operations.SaveProtographOperation);
var RC = Packages.com.google.refine.model.recon.ReconConfig;
RC.registerReconConfig(module, "strict", Packages.com.google.refine.freebase.model.recon.StrictReconConfig);
RC.registerReconConfig(module, "extend", Packages.com.google.refine.freebase.model.recon.DataExtensionReconConfig);
var ER = Packages.com.google.refine.exporters.ExporterRegistry;
ER.registerExporter("tripleloader", new Packages.com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter());
ER.registerExporter("mqlwrite", new Packages.com.google.refine.freebase.ProtographTransposeExporter.MqlwriteLikeExporter());
var FCR = Packages.com.google.refine.grel.ControlFunctionRegistry;
FCR.registerFunction("mqlKeyQuote", new Packages.com.google.refine.freebase.expr.MqlKeyQuote());
FCR.registerFunction("mqlKeyUnquote", new Packages.com.google.refine.freebase.expr.MqlKeyUnquote());
Packages.com.google.refine.model.Project.
registerOverlayModel("freebaseProtograph", Packages.com.google.refine.freebase.protograph.Protograph);
ClientSideResourceManager.addPaths(
"project/scripts",
module,
[
"scripts/extension.js",
"scripts/util/freebase.js",
"scripts/dialogs/freebase-loading-dialog.js",
"scripts/dialogs/extend-data-preview-dialog.js",
"scripts/dialogs/schema-alignment/dialog.js",
"scripts/dialogs/schema-alignment/ui-node.js",
"scripts/dialogs/schema-alignment/ui-link.js"
]
);
ClientSideResourceManager.addPaths(
"project/styles",
module,
[
"styles/dialogs/freebase-loading-dialog.less",
"styles/dialogs/extend-data-preview-dialog.less",
"styles/dialogs/schema-alignment-dialog.less"
]
);
}

BIN
extensions/freebase/module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/libsrc/google-api-services-freebase-v1-rev25-1.13.2-beta-sources.jar
View File

Binary file not shown.

4
extensions/freebase/module/MOD-INF/module.properties
View File

@ -1,4 +0,0 @@
name = freebase
description = OpenRefine Freebase Extension
templating = false
requires = core

125
extensions/freebase/module/langs/translation-default.json
View File

@ -1,125 +0,0 @@
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it.",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Set Freebase API Key",
"align-schema": "Align to Freebase's schemas...",
"load": "Load into Freebase...",
"browse-data-load": "Browse data load details...",
"import-qa": "Import QA data",
"add-columns": "Add columns from Freebase ...",
"warning-load": "You have not tried to load the data in this project into Freebase yet."
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}

125
extensions/freebase/module/langs/translation-en.json
View File

@ -1,125 +0,0 @@
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it.",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Set Freebase API Key",
"align-schema": "Align to Freebase's schemas...",
"load": "Load into Freebase...",
"browse-data-load": "Browse data load details...",
"import-qa": "Import QA data",
"add-columns": "Add columns from Freebase ...",
"warning-load": "You have not tried to load the data in this project into Freebase yet."
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}

125
extensions/freebase/module/langs/translation-fr.json
View File

@ -1,125 +0,0 @@
{
"fb-schema-alignment": {
"close-confirm": "Des modifications n’ont pas été enregistrées. Fermer malgré tout ?",
"status-warning": "Des modifications n’ont pas été enregistrées.",
"assert-link-found": "Considérer comme lien lorsqu’une cellule de cette colonne contient 'true'",
"search-pick-property": "Rechercher une propriété ou en choisir une plus bas",
"search-property": "Rechercher une propriété",
"cell": "cellule",
"cells": "cellules",
"which-column": "Quelle colonne ?",
"configure": "Configuration...",
"which-topic": "Quel sujet ?",
"what-value": "Quelle valeur ?",
"anonymous": "anonyme",
"add-property": "ajouter une propriété",
"anonymous-node": "Nœud anonyme",
"freebase-topic": "Sujet Freebase",
"value": "Valeur",
"skeleton-node": "Nœud du squelette du schéma d’alignement",
"text": "texte",
"int": "entier",
"float": "flottant",
"double": "double",
"boolean": "booléen",
"date-time": "date/heure",
"rawstring": "simple chaîne",
"set-to-cell": "Indiquer Set to Cell in Column",
"cell-content-used": "Le contenu de la cellule est utilisé...",
"specify-fb-topic": "pour préciser le sujet de Freebase, tel que réconcilié",
"type-new-topics": "Spécifier les nouveau sujets comme",
"literal-value": "comme une valeur littérale",
"literal-type": "Type littéral",
"text-language": "Langue du texte",
"key-namespace": "comme une clé dans un espace de noms",
"namespace": "Espace de noms",
"generate-anonymous": "Créer un nœud de graphe anonyme",
"assign-type": "Assigner un type au nœud",
"use-existing-topic": "Utiliser un sujet existant de Freebase",
"value-type": "Type de valeur",
"language": "Langue",
"use-literal-value": "Utiliser une valeur littérale",
"column-warning": "Vous devez choisir au moins une colonne",
"new-node-warning": "Pour créer un nouveau nœud du graphe, vous devez indiquer son type.",
"namespace-warning": "Merci d’indiquer l’espace de noms.",
"anonymous-node-warning": "Pour créer un nœud de graphe anonyme, vous devez spécifier son type.",
"specify-topic-warning": "Merci d’indiquer le sujet à utiliser qui existe dans Freebase",
"specify-value-warning": "Merci d’indiquer la valeur à utiliser"
},
"fb-interface": {
"dialog-header": "Aligner sur les schémas de Freebase",
"body-text": "Le squelette d’alignement du schéma ci-dessous indique comment vos grilles de données seront transformées en graphes de données dans les schémas Freebase.",
"find-more": "En savoir plus...",
"skeleton": "Squelette",
"mql-preview": "Aperçu MQL",
"tripleloader-preview": "Aperçu du chargeur de triplet"
},
"fb-dialogs": {
"sign-in": "Se connecter à Freebase",
"enable-loading": "pour activer le chargement",
"error-new-topic": "Erreur lors de la création du nouveau sujet",
"error-loading-data": "Erreur lors du chargement des données",
"add-info-source": "Cliquer ici pour ajouter une nouvelle source d’informations",
"dialog-header": "Charger les données dans Freebase",
"no-triples-dataset": "Cet ensemble de données n’a pas de triplet",
"warning-aligned": "L’avez-vous déjà aligné avec les schémas Freebase ?",
"name-of-data": "Nom de votre chargement de données",
"source-id": "ID Source (facultatif)",
"bodytext-1": "Note : Vos donnes seront chargées seulement dans",
"bodytext-2": "Le bac à sable est l’endroit où chacun peut expérimenter les technologies de Freebase sans perturber la base officielle",
"bodytext-3": "Le bac à sable a récupéré",
"sandbox-link": "Bac à sable",
"freebase-link": "Freebase",
"refreshed-link": "rafraîchir périodiquement",
"bodytext-4": "De façon à charger vos données dans la base officielle Freebase, vous devez d’abord les charger dans le bac à sable. Ensuite, elles sont vérifiées par une procédure d’assurance qualité (AQ) avant de pouvoir être chargées proprement.",
"quality-assurance": "Assurance qualité",
"bodytext-5": "Après le chargement dans le bac à sable, After loaded into Sandbox, demandez l’aide d’autres personnes pour procéder à une double vérification de la qualité des données chargées de façon à pouvoir les charger dans Freebase.",
"triple-schedule": "chargement des triplets correctement planifié",
"follow-progress": "Suivre la progression du chargement dans la",
"refinery-link": "Raffinerie Freebase",
"signed-as": "Connecté comme :",
"sign-out": "Se déconnecter"
},
"fb-qa": {
"header": "Charger des données AQ ?",
"bodytext-1": "D’autres personnes seront chargées de procéder à une double vérification des données que vous aurez versées afin de garantir leur qualité. Leur temps et leur travail ont un coût.",
"bodytext-2": "Vous devez prendre vous-même toutes les mesures raisonnables pour élimiter les erreurs des données que vous chargez. Votre prudence sera grandement appréciée.",
"tell-more": "M’en dire plus...",
"ok-button": "Oui, charger les données AQ"
},
"fb-extend": {
"add-column": "Ajouter les colonnes de Freebase sur la base de la colonne",
"warning-add-properties": "Merci d’ajouter des propriétés d’abord.",
"querying-freebase": "Requête sur Freebase ...",
"remove-column": "Supprimer cette colonne",
"add-constraints": "Ajouter des contraintes à cette colonne",
"mql-constraints": "Indiquer les contraintes de la requête MQL en JSON",
"warning-valid-json": "Merci de vérifier que le code JSON que vous avez entré est valide.",
"warning-json-obj": "Le code JSON que vous indiquez doit être un objet, c’est-à-dire être de la forme { ... }.",
"add-property": "Ajouter une propriété",
"suggested-properties": "Propriétés suggérées",
"constraint": "Contrainte"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Indiquer la clé de l’API Freebase",
"align-schema": "Alignement sur les schémas de Freebase...",
"load": "Chargement dans Freebase...",
"browse-data-load": "Consulter le détail du chargement des données...",
"import-qa": "Importer des données AQ",
"add-columns": "Ajouter des colonnes à partir de Freebase...",
"warning-load": "Vous n’avez pas encore essayé de charger les données de ce projet dans Freebase."
},
"fb-buttons": {
"save": "Sauver",
"save-load": "Sauver & charger",
"close": "Fermer",
"reset": "Réinitialiser",
"cancel": "Annuler",
"align-now": "Aligner maintenant",
"settings": "Paramètres",
"preview": "Aperçu",
"load-sandbox": "Charger dans un bac à sable",
"ok": "Ok"
}
}

125
extensions/freebase/module/langs/translation-it.json
View File

@ -1,125 +0,0 @@
{
"fb-schema-alignment": {
"close-confirm": "Ci sono cambiamenti non salvati. Chiudere comunque?",
"status-warning": "Ci sono cambiamenti non salvati.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Cerca una proprietà o scegline una dalle seguenti",
"search-property": "Cerca una proprietà",
"cell": "cella",
"cells": "celle",
"which-column": "Quale colonna?",
"configure": "Configura...",
"which-topic": "Quale topic?",
"what-value": "Quale valore?",
"anonymous": "anonimo",
"add-property": "aggiungi proprietà",
"anonymous-node": "Nodo Anonimo",
"freebase-topic": "Topic Freebase",
"value": "Valore",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "testo",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "data/ora",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "Il contenuto della cella è usato ...",
"specify-fb-topic": "per specificare un topic Freebase, come riconciliato",
"type-new-topics": "Inserisci il nuovo topic come",
"literal-value": "come un valore letterale",
"literal-type": "Tipo letterale",
"text-language": "Lingua di testo",
"key-namespace": "come una chiave nel namespace",
"namespace": "Namespace",
"generate-anonymous": "Genera un nodo anonimo nel grafo",
"assign-type": "Assegna un tipo al nodo",
"use-existing-topic": "Usa un topic Freebase esistente",
"value-type": "Tipo valore",
"language": "Lingua",
"use-literal-value": "Usa un valore letterale",
"column-warning": "Devi selezionare almeno una colonna",
"new-node-warning": "Per creare un nuovo nodo nel grafo, devi specificarne un tipo.",
"namespace-warning": "Specifica il namespace.",
"anonymous-node-warning": "Per generare un nodo anonimo nel grafo, devi specificarne un tipo.",
"specify-topic-warning": "Per favore specifica quale topic esistente di Freebase usare",
"specify-value-warning": "Per favore specifica il valore da usare"
},
"fb-interface": {
"dialog-header": "Allinea con gli schemi Freebase",
"body-text": "Lo scheletro per l'allineamento della schema specifica come i tuoi dati in formato tabellare saranno trasformati in un formato a grafo con lo schema di Freebase.",
"find-more": "Per saperne di più ...",
"skeleton": "Scheletro",
"mql-preview": "Anteprima MQL-like",
"tripleloader-preview": "Anteprima TripleLoader"
},
"fb-dialogs": {
"sign-in": "Accedi a Freebase",
"enable-loading": "per abilitare il caricamento",
"error-new-topic": "Errore nella creazione di un nuovo topic",
"error-loading-data": "Errore durante il caricamento dei dati",
"add-info-source": "Clicca qui per aggiungere una nuova sorgente di informazioni",
"dialog-header": "Carica i dati in Freebase",
"no-triples-dataset": "Questo dataset non ha triple",
"warning-aligned": "Hai già effettuato l'allineamento con lo schema Freebase?",
"name-of-data": "Nome del caricamento dati",
"source-id": "ID sorgente (opzionale)",
"bodytext-1": "Nota: i tuoi dati saranno caricati solamente in",
"bodytext-2": "Sandbox è dove chiunque può sperimentare le tecnologie Freebase senza creare danni all'ufficiale",
"bodytext-3": "Sandbox viene",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " aggiornata periodicamente",
"bodytext-4": "Per caricare i tuoi data su Freebase ufficiale, devi prima caricarli nel Sandbox. Successivamente, devono passare un processo di analisi qualitativa prima di essere caricati su Freebase.",
"quality-assurance": "Analisi qualitativa",
"bodytext-5": "Dopo essere caricati nella Sandbox, viene chiesto l'aiuto di altre persone per controllare due volte la qualità di questi dati in modo da poter poi essere caricati in Freebase.",
"triple-schedule": "triple pianificate con successo per il caricamento",
"follow-progress": "Segui il processo di caricamento nel ",
"refinery-link": "Freebase Refinery",
"signed-as": "Accesso effettuato come:",
"sign-out": "Esci"
},
"fb-qa": {
"header": "Caricamento dati con analisi qualitativa?",
"bodytext-1": "Alcune persone verranno incaricate per controllare i tuoi dati allo scopo di verificarne la qualità. Il loro tempo e lavoro ha un costo.",
"bodytext-2": "Tu stesso dovresti aver già effettuato la maggior parte dei controlli per assicurarti di aver rimosso errori dai dati. La tua prudenza è enormemente apprezzata.",
"tell-more": "Dimmi di più ...",
"ok-button": "Si, Caricamento dati con analisi qualitativa"
},
"fb-extend": {
"add-column": "Aggiungi colonne da Freebase basandoti sulla colonna",
"warning-add-properties": "Prima aggiungi delle proprietà.",
"querying-freebase": "Interrogando Freebase ...",
"remove-column": "Rimuovi questa colonna",
"add-constraints": "Aggiungi vincoli a questa colonna",
"mql-constraints": "Inserisci i vincoli per la query MQL come JSON",
"warning-valid-json": "Assicurati che l'oggetto JSON inserito sia valido.",
"warning-json-obj": "Il JSON che inserisci deve essere un oggetto, cioè in questa forma: { ... }.",
"add-property": "Aggiungi Proprietà",
"suggested-properties": "Proprietà suggerite",
"constraint": "Vincoli"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Imposta l'API Key Freebase",
"align-schema": "Allinea con lo schema Freebase...",
"load": "Carica in Freebase...",
"browse-data-load": "Vedi i dettagli per il caricamento dati...",
"import-qa": "Importa dati con controllo qualità",
"add-columns": "Aggiungi colonne da Freebase ...",
"warning-load": "Non hai ancora provato a caricare i dati di questo progetto in Freebase."
},
"fb-buttons": {
"save": "Salva",
"save-load": "Salva & Carica",
"close": "Chiudi",
"reset": "Reset",
"cancel": "Cancella",
"align-now": "Allinea adesso",
"settings": "Settings",
"preview": "Anteprima",
"load-sandbox": "Carica nella Sandbox",
"ok": "Ok"
}
}

12
extensions/freebase/module/scripts/dialogs/confirm-qa-dialog.html
View File

@ -1,12 +0,0 @@
<div class="dialog-frame" style="width: 400px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<p class="body-text" bind="bodytext1"></p>
<p class="body-text" bind="bodytext2"></p>
<p><a href="http://wiki.freebase.com/wiki/Refinery" target="_blank" bind="tellMore"></a></p>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button class="button" bind="okButton"></button>
<button class="button" bind="cancelButton"></button>
</div>
</div>

26
extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html
View File

@ -1,26 +0,0 @@
<div class="dialog-frame extend-data-preview-dialog" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div class="grid-layout layout-normal layout-full"><table rows="4">
<tr>
<td width="300" height="1" ><span bind="fb_add_property"></span></td>
<td height="1">Preview</td>
<td height="1" width="1%"><button class="button" bind="resetButton"></button></td>
</tr>
<tr>
<td style="vertical-align: top;" height="1"><div class="input-container"><input bind="addPropertyInput" /></div></td>
<td style="vertical-align: top;" rowspan="3" colspan="2"><div class="preview-container" bind="previewContainer"></div></td>
</tr>
<tr>
<td height="1" bind="suggested_properties"></td>
</tr>
<tr>
<td><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>
</tr>
</table></div>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button class="button" bind="okButton"></button>
<button class="button" bind="cancelButton"></button>
</div>
</div>

402
extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js
View File

@ -1,402 +0,0 @@
/*
Copyright 2010,2012 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
function ExtendDataPreviewDialog(column, columnIndex, rowIndices, onDone) {
this._column = column;
this._columnIndex = columnIndex;
this._rowIndices = rowIndices;
this._onDone = onDone;
this._extension = { properties: [] };
var self = this;
this._dialog = $(DOM.loadHTML("freebase", "scripts/dialogs/extend-data-preview-dialog.html"));
this._elmts = DOM.bind(this._dialog);
this._elmts.dialogHeader.text($.i18n._('fb-extend')["add-column"]);
this._elmts.fb_add_property.html($.i18n._('fb-extend')["add-property"]);
this._elmts.suggested_properties.text($.i18n._('fb-extend')["suggested-properties"]);
this._elmts.resetButton.text($.i18n._('fb-buttons')["reset"]);
this._elmts.okButton.html('&nbsp;&nbsp;'+$.i18n._('fb-buttons')["ok"]+'&nbsp;&nbsp;');
this._elmts.cancelButton.text($.i18n._('fb-buttons')["cancel"]);
this._elmts.resetButton.click(function() {
self._extension.properties = [];
self._update();
});
this._elmts.okButton.click(function() {
if (self._extension.properties.length === 0) {
alert($.i18n._('fb-extend')["warning-add-properties"]);
} else {
DialogSystem.dismissUntil(self._level - 1);
self._onDone(self._extension);
}
});
this._elmts.cancelButton.click(function() {
DialogSystem.dismissUntil(self._level - 1);
});
var dismissBusy = DialogSystem.showBusy();
var type = (column.reconConfig) && (column.reconConfig.type) ? column.reconConfig.type.id : "/common/topic";
ExtendDataPreviewDialog.getAllProperties(type, function(properties) {
dismissBusy();
self._show(properties);
});
}
ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
var done = false;
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?type=" + typeID + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
var allProperties = [];
for (var i = 0; i < data.properties.length; i++) {
var property = data.properties[i];
var property2 = {
id: property.id,
name: property.name
};
if ("id2" in property) {
property2.expected = property.schema2;
property2.properties = [{
id: property.id2,
name: property.name2,
expected: property.expects
}];
} else {
property2.expected = property.expects;
}
allProperties.push(property2);
}
allProperties.sort(function(a, b) { return a.name.localeCompare(b.name); });
onDone(allProperties);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
ExtendDataPreviewDialog.prototype._show = function(properties) {
this._level = DialogSystem.showDialog(this._dialog);
var n = this._elmts.suggestedPropertyContainer.offset().top +
this._elmts.suggestedPropertyContainer.outerHeight(true) -
this._elmts.addPropertyInput.offset().top;
this._elmts.previewContainer.height(Math.floor(n));
var self = this;
var container = this._elmts.suggestedPropertyContainer;
var renderSuggestedProperty = function(property) {
var label = ("properties" in property) ? (property.name + " &raquo; " + property.properties[0].name) : property.name;
var div = $('<div>').addClass("suggested-property").appendTo(container);
$('<a>')
.attr("href", "javascript:{}")
.html(label)
.appendTo(div)
.click(function() {
self._addProperty(property);
});
};
for (var i = 0; i < properties.length; i++) {
renderSuggestedProperty(properties[i]);
}
var suggestConfig = {
type: '/type/property', // NOTE: requires patched Suggest to pass this through
// Default returns id, lang, mid, name, notable {id,name}, score
mql_output : JSON.stringify({'name':null,'id':null,'mid':null, '/type/property/expected_type':{'name':null,'id':null}}),
};
if ((this._column.reconConfig) && (this._column.reconConfig.type)) {
suggestConfig.filter = '(should (any namespace:/type/object namespace:/common/topic namespace:' + this._column.reconConfig.type.id + '))';
}
this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
self._addProperty({
id : data.id,
name: data.name,
expected: data["/type/property/expected_type"]
});
});
};
ExtendDataPreviewDialog.prototype._update = function() {
this._elmts.previewContainer.empty().text($.i18n._('fb-extend')["querying-freebase"]);
var self = this;
var params = {
project: theProject.id,
columnName: this._column.name
};
$.post(
"command/freebase/preview-extend-data?" + $.param(params),
{
rowIndices: JSON.stringify(this._rowIndices),
extension: JSON.stringify(this._extension)
},
function(data) {
self._renderPreview(data);
},
"json"
);
};
ExtendDataPreviewDialog.prototype._addProperty = function(p) {
var addSeveralToList = function(properties, oldProperties) {
for (var i = 0; i < properties.length; i++) {
addToList(properties[i], oldProperties);
}
};
var addToList = function(property, oldProperties) {
for (var i = 0; i < oldProperties.length; i++) {
var oldProperty = oldProperties[i];
if (oldProperty.id == property.id) {
if ("included" in property) {
oldProperty.included = "included" in oldProperty ?
(oldProperty.included || property.included) :
property.included;
}
if ("properties" in property) {
if ("properties" in oldProperty) {
addSeveralToList(property.properties, oldProperty.properties);
} else {
oldProperty.properties = property.properties;
}
}
return;
}
}
oldProperties.push(property);
};
addToList(p, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
var self = this;
var container = this._elmts.previewContainer.empty();
if (data.code == "error") {
container.text("Error.");
return;
}
var table = $('<table>')[0];
var trHead = table.insertRow(table.rows.length);
$('<th>').appendTo(trHead).text(this._column.name);
var renderColumnHeader = function(column) {
var th = $('<th>').appendTo(trHead);
$('<span>').html(column.names.join(" &raquo; ")).appendTo(th);
$('<br>').appendTo(th);
$('<a href="javascript:{}"></a>')
.text("remove")
.addClass("action")
.attr("title", $.i18n._('fb-extend')["remove-column"])
.click(function() {
self._removeProperty(column.path);
}).appendTo(th);
$('<a href="javascript:{}"></a>')
.text("constrain")
.addClass("action")
.attr("title", $.i18n._('fb-extend')["add-constraints"])
.click(function() {
self._constrainProperty(column.path);
}).appendTo(th);
};
for (var c = 0; c < data.columns.length; c++) {
renderColumnHeader(data.columns[c]);
}
for (var r = 0; r < data.rows.length; r++) {
var tr = table.insertRow(table.rows.length);
var row = data.rows[r];
for (var c = 0; c < row.length; c++) {
var td = tr.insertCell(tr.cells.length);
var cell = row[c];
if (cell !== null) {
if ($.isPlainObject(cell)) {
$('<a>').attr("href", "http://www.freebase.com/view" + cell.id).text(cell.name).appendTo(td);
} else {
$('<span>').text(cell).appendTo(td);
}
}
}
}
container.append(table);
};
ExtendDataPreviewDialog.prototype._removeProperty = function(path) {
var removeFromList = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
if ("included" in property) {
delete property.included;
}
} else if ("properties" in property && property.properties.length > 0) {
removeFromList(path, index + 1, property.properties);
}
if (!("properties" in property) || property.properties.length === 0) {
properties.splice(i, 1);
}
return;
}
}
};
removeFromList(path, 0, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._findProperty = function(path) {
var find = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
return property;
} else if ("properties" in property && property.properties.length > 0) {
return find(path, index + 1, property.properties);
}
break;
}
}
return null;
};
return find(path, 0, this._extension.properties);
};
ExtendDataPreviewDialog.prototype._constrainProperty = function(path) {
var self = this;
var property = this._findProperty(path);
var frame = DialogSystem.createDialog();
frame.width("500px");
var header = $('<div></div>').addClass("dialog-header").text(" " + path.join(" > ")).appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
body.html(
'<div class="grid-layout layout-normal layout-full"><table>' +
'<tr><td>' +
$.i18n._('fb-extend')["mql-constraints"] +
'</td></tr>' +
'<tr><td>' +
'<textarea style="width: 100%; height: 300px; font-family: monospace;" bind="textarea"></textarea>' +
'</td></tr>' +
'</table></div>'
);
var bodyElmts = DOM.bind(body);
if ("constraints" in property) {
bodyElmts.textarea[0].value = JSON.stringify(property.constraints, null, 2);
} else {
bodyElmts.textarea[0].value = JSON.stringify({ "limit" : 10 }, null, 2);
}
footer.html(
'<button class="button" bind="okButton">&nbsp;&nbsp;'+$.i18n._('fb-buttons')["ok"]+'&nbsp;&nbsp;</button>' +
'<button class="button" bind="cancelButton">'+$.i18n._('fb-buttons')["cancel"]+'</button>'
);
var footerElmts = DOM.bind(footer);
var level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(level - 1);
};
footerElmts.cancelButton.click(dismiss);
footerElmts.okButton.click(function() {
try {
var o = JSON.parse(bodyElmts.textarea[0].value);
if (o === undefined) {
alert($.i18n._('fb-extend')["warning-valid-json"]);
return;
}
if ($.isArray(o) && o.length == 1) {
o = o[0];
}
if (!$.isPlainObject(o)) {
alert($.i18n._('fb-extend')["warning-json-obj"]);
return;
}
property.constraints = o;
dismiss();
self._update();
} catch (e) {
//console.log(e);
}
});
bodyElmts.textarea.focus();
};

67
extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html
View File

@ -1,67 +0,0 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div bind="unalignedCase" style="display: none;" class="freebase-loading-tripleloader-message">
<h2 bind="no_triples_dataset"></h2>
<p bind="warning_aligned"></p><button class="button" bind="alignButton"></button>
</div>
<div bind="functionalCase" style="display: none;">
<div class="refine-tabs" bind="functionalTabs">
<ul>
<li><a href="#freebase-loading-dialog-functional-tabs-settings" bind="settingsBtn"></a></li>
<li><a href="#freebase-loading-dialog-functional-tabs-preview" bind="previewBtn"></a></li>
</ul>
<div id="freebase-loading-dialog-functional-tabs-settings"
class="freebase-loading-dialog-functional-tab-panel"><div class="grid-layout layout-normal"><table>
<tr><td width="20%" bind="name_of_data"></td>
<td colspan="2"><input type="text" size="40" id="freebase-loading-source-name" bind="source_name" /></td>
</tr>
<tr><td bind="source_id_label"></td>
<td colspan="2"><input type="text" size="60" id="freebase-loading-source-id" bind="source_id" /></td>
</tr>
<tr>
<td colspan="3">
<p class="body-text"><span bind="bodytext1"></span>
<a href="http://www.sandbox-freebase.com/" target="_blank" bind="sandbox_link"></a>.
<span bind="bodytext2"> </span><a href="http://www.freebase.com/" bind="freebase_link"></a>.
<span bind="bodytext3"></span><a href="http://wiki.freebase.com/wiki/Sandbox"
target="_blank" bind="refreshed_link"></a>.
</p>
<p class="body-text" bind="bodytext4">
<a href="http://wiki.freebase.com/wiki/Refinery" target="_blank" bind="findOutMore"></a>
</p>
</td>
</tr>
<tr><td bind="quality_assurance"></td>
<td><input bind="qaCheckbox" type="checkbox" name="qa" value="qa" /></td>
<td bind="bodytext5"></td>
</tr>
</table></div></div>
<div id="freebase-loading-dialog-functional-tabs-preview"
class="freebase-loading-dialog-functional-tab-panel freebase-loading-tripleloader-data"
style="display: none;"
bind="previewContainer">
</div>
</div>
</div>
<div bind="loadedCase" style="display: none;" class="freebase-loading-tripleloader-message">
<h2><span bind="tripleCountSpan"></span> <span bind="triple_success_load"></span></h2>
<h4><span bind="follow_progress"></span> <a bind="refineryLink" target="_new"></a>.</h4>
</div>
<div bind="errorCase" style="display: none;" class="freebase-loading-tripleloader-message">
<h2 bind="errorMessage"></h2>
<div bind="errorDetails"></div>
</div>
</div>
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-normal layout-full"><table><tr>
<td bind="left" style="text-align: left" width="60%" nowrap="true">
<button class="button" bind="cancelButton" id="freebase-loading-cancel"></button>
</td>
<td bind="center" style="text-align: right" width="20%" nowrap="true">
<div bind="authorization" class="freebase-loading-authorization"></div>
</td>
<td bind="right" style="text-align: right" width="20%" nowrap="true">
<button class="button button-primary button-disabled" bind="loadButton" id="freebase-loading-load" disabled></button>
</td>
</tr></table></div></div>
</div>

286
extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js
View File

@ -1,286 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
function FreebaseLoadingDialog() {
this._createDialog();
this._signedin = false;
}
FreebaseLoadingDialog.prototype._createDialog = function() {
var self = this;
var dialog = $(DOM.loadHTML("freebase", "scripts/dialogs/freebase-loading-dialog.html"));
this._elmts = DOM.bind(dialog);
this._elmts.cancelButton.click(function() { self._dismiss(); });
var provider = "freebase.com";
var authorization = this._elmts.authorization;
var loadButton = this._elmts.loadButton;
this._elmts.dialogHeader.text($.i18n._('fb-dialogs')["dialog-header"]);
this._elmts.no_triples_dataset.text($.i18n._('fb-dialogs')["no-triples-dataset"]);
this._elmts.warning_aligned.text($.i18n._('fb-dialogs')["warning-aligned"]);
this._elmts.alignButton.text($.i18n._('fb-buttons')["align-now"]);
this._elmts.settingsBtn.text($.i18n._('fb-buttons')["settings"]);
this._elmts.previewBtn.text($.i18n._('fb-buttons')["preview"]);
this._elmts.name_of_data.text($.i18n._('fb-dialogs')["name-of-data"]);
this._elmts.source_id_label.text($.i18n._('fb-dialogs')["source-id"]);
this._elmts.bodytext1.text($.i18n._('fb-dialogs')["bodytext-1"]);
this._elmts.bodytext2.text($.i18n._('fb-dialogs')["bodytext-2"]);
this._elmts.bodytext3.text($.i18n._('fb-dialogs')["bodytext-3"]);
this._elmts.sandbox_link.text($.i18n._('fb-dialogs')["sandbox-link"]);
this._elmts.freebase_link.text($.i18n._('fb-dialogs')["freebase-link"]);
this._elmts.refreshed_link.text($.i18n._('fb-dialogs')["refreshed-link"]);
this._elmts.bodytext4.text($.i18n._('fb-dialogs')["bodytext-4"]);
this._elmts.findOutMore.text($.i18n._('fb-interface')["find-more"]);
this._elmts.quality_assurance.text($.i18n._('fb-dialogs')["quality-assurance"]);
this._elmts.bodytext5.text($.i18n._('fb-dialogs')["bodytext-5"]);
this._elmts.triple_success_load.text($.i18n._('fb-dialogs')["triple-schedule"]);
this._elmts.follow_progress.text($.i18n._('fb-dialogs')["follow-progress"]);
this._elmts.refineryLink.text($.i18n._('fb-dialogs')["refinery-link"]);
this._elmts.cancelButton.text($.i18n._('fb-buttons')["cancel"]);
this._elmts.loadButton.text($.i18n._('fb-buttons')["load-sandbox"]);
// TODO: Does this need to be redone for the new API / freebase.com site?
var check_authorization = function(cont) {
var check_authorization_url = "command/freebase/check-authorization/" + provider;
$.get(check_authorization_url, function(data) {
if ("status" in data && data.code == "/api/status/ok") {
authorization.html($.i18n._('fb-dialogs')["signed-as"]+' <a target="_new" href="http://www.freebase.com/view/user/' + data.username + '">' + data.username + '</a> | <a href="javascript:{}" bind="signout">'+$.i18n._('fb-dialogs')["sign-out"]+'</a>').show();
DOM.bind(authorization).signout.click(function() {
self._signedin = false;
loadButton.attr("disabled","disabled");
$("#freebase-loading-graph-selector-freebase").attr("disabled","disabled").button("refresh");
Sign.signout(function() {
$('.signedin').hide();
$('.signedout').show();
check_authorization();
},provider);
});
loadButton.unbind().click(function() {
self._load();
});
self._signedin = true;
$("#freebase-loading-source-name").keyup();
if (typeof cont == "function") cont(data);
} else {
authorization.html('<a href="javascript:{}" bind="signin">'+$.i18n._('fb-dialogs')["sign-in"]+'</a> '+$.i18n._('fb-dialogs')["enable-loading"]).show();
self._signedin = false;
$("#freebase-loading-source-name").keyup();
DOM.bind(authorization).signin.click(function() {
Sign.signin(function() {
$('.signedout').hide();
$('.signedin').show();
check_authorization(cont);
},provider,check_authorization_url);
});
}
},"json");
};
var make_topic = function(new_topic_id, topic_type, cont) {
var mql_query = [{
"create": "unless_exists",
"name": new_topic_id,
"a:type": topic_type,
"b:type": "/common/topic",
"id": null,
"guid": null
}];
$.post("command/freebase/mqlwrite/" + provider,
{ "query" : JSON.stringify(mql_query) },
function(data) {
if ("status" in data && data.code == "/api/status/ok") {
self._elmts.source_id.val(data.result[0].id);
if (typeof cont == "function") cont();
} else {
self._show_error($.i18n._('fb-dialogs')["error-new-topic"], data);
}
},
"json"
);
};
var show_triples = function(cont) {
$.post(
"command/freebase/preview-protograph?" + $.param({ project: theProject.id }),
{
protograph: JSON.stringify(theProject.overlayModels.freebaseProtograph || {}),
engine: JSON.stringify(ui.browsingEngine.getJSON())
},
function(data) {
if ("tripleloader" in data) {
self._elmts.functionalCase.show();
self._level = DialogSystem.showDialog(dialog);
self._elmts.functionalTabs.tabs();
self._elmts.previewContainer.text(data.tripleloader).show();
self._elmts.source_name.keyup(function() {
if (self._signedin && $(this).val() != "") {
loadButton.removeAttr("disabled").removeClass("button-disabled");
} else {
loadButton.attr("disabled","disabled").addClass("button-disabled");
}
});
self._elmts.source_id.suggest({
filter: "(all type:/dataworld/information_source)",
suggest_new: $.i18n._('fb-dialogs')["add-info-source"]
}).bind("fb-select", function(e, data) {
self._elmts.source_id.val(data.id);
}).bind("fb-select-new", function(e, val) {
make_topic(val, "/dataworld/information_source");
});
$.getJSON(
"command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobName" }),
null,
function(data) {
self._elmts.source_name[0].value = (data.value) ? data.value : theProject.metadata.name;
}
);
if (typeof cont == "function") cont();
} else {
self._elmts.unalignedCase.show();
self._level = DialogSystem.showDialog(dialog);
self._elmts.alignButton.click(function() {
self._dismiss();
FreebaseExtension.handlers.editSchemaAlignment(false);
});
self._end();
}
},
"json"
);
};
show_triples(check_authorization);
};
FreebaseLoadingDialog.prototype._load = function() {
var self = this;
var qa = self._elmts.qaCheckbox.is(':checked');
var get_refinery_url = function(url) {
return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0];
};
var doLoad = function() {
var dismissBusy = DialogSystem.showBusy();
$.post(
"command/freebase/upload-data",
{
"project" : theProject.id,
"qa" : qa,
"engine" : JSON.stringify(ui.browsingEngine.getJSON()),
"source_name" : self._elmts.source_name.val(),
"source_id" : self._elmts.source_id.val()
},
function(data) {
dismissBusy();
var body = self._elmts.dialogBody;
if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) {
self._elmts.tripleCountSpan.text(data.result.added);
self._elmts.refineryLink.attr("href", get_refinery_url(data.result.status_url));
self._elmts.functionalCase.hide();
self._elmts.loadedCase.show();
self._end();
} else {
self._show_error($.i18n._('fb-dialogs')["error-loading-data"],data);
}
},
"json"
);
};
if (qa) {
var dialog = $(DOM.loadHTML("freebase", "scripts/dialogs/confirm-qa-dialog.html"));
var elmts = DOM.bind(dialog);
var level = DialogSystem.showDialog(dialog);
var dismiss = function() {
DialogSystem.dismissUntil(level - 1);
};
this._elmts.refineryLink.text($.i18n._('fb-qa')["header"]);
this._elmts.bodytext1.text($.i18n._('fb-qa')["bodytext-1"]);
this._elmts.bodytext2.text($.i18n._('fb-qa')["bodytext-2"]);
this._elmts.tellMore.text($.i18n._('fb-qa')["tell-more"]);
this._elmts.okButton.text($.i18n._('fb-qa')["ok-button"]);
this._elmts.cancelButton.text($.i18n._('fb-qa')["ok-button"]);
elmts.okButton.click(function() {
doLoad();
dismiss();
});
elmts.cancelButton.click(function() {
dismiss();
});
} else {
doLoad();
}
};
FreebaseLoadingDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
FreebaseLoadingDialog.prototype._show_error = function(msg, error) {
this._elmts.dialogBody.children().hide();
this._elmts.errorCase.show();
this._elmts.errorMessage.text(msg);
this._elmts.errorDetails.html(
(('message' in error) ? '<p>' + error.message + '</p>' : '<pre>' + JSON.stringify(error, null, 2) + '</pre>') +
(('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</pre>' : "")
);
this._end();
console.log(error);
};
FreebaseLoadingDialog.prototype._end = function() {
var self = this;
this._elmts.loadButton.text("Close").removeAttr("disabled").removeClass("button-disabled").unbind().click(function() {
self._dismiss();
});
this._elmts.cancelButton.hide();
this._elmts.authorization.hide();
};

324
extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js
View File

@ -1,324 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
var SchemaAlignment = {};
SchemaAlignment.autoAlign = function() {
var protograph = {};
var columns = theProject.columnModel.columns;
var typedCandidates = [];
var candidates = [];
for (var c = 0; c < columns.length; c++) {
var column = columns[c];
var typed = (column.reconConfig) &&
ReconciliationManager.isFreebaseIdOrMid(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace);
var candidate = {
status: "unbound",
typed: typed,
index: c,
column: column
};
candidates.push(candidate);
if (typed) {
typedCandidates.push(candidate);
}
}
if (typedCandidates.length > 0) {
} else {
var queries = {};
for (var i = 0; i < candidates.length; i++) {
var candidate = candidates[i];
var name = SchemaAlignment._cleanName(candidate.column.name);
var key = "t" + i + ":search";
queries[key] = {
"query" : name,
"limit" : 10,
"type" : "/type/type,/type/property",
"type_strict" : "any"
};
}
SchemaAlignment._batchSearch(queries, function(result) {
console.log(result);
});
}
};
SchemaAlignment._batchSearch = function(queries, onDone) {
var keys = [];
for (var n in queries) {
if (queries.hasOwnProperty(n)) {
keys.push(n);
}
}
var result = {};
var args = [];
var makeBatch = function(keyBatch) {
var batch = {};
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
batch[key] = queries[key];
}
// TODO: New API doesn't accept multiple queries
// args.push("https://www.googleapis.com/freebase/v1/search?key=" + Freebase.API_KEY + "&" +
args.push("http://api.freebase.com/api/service/search?" + // FIXME:
$.param({ "queries" : JSON.stringify(batch) }) + "&callback=?");
args.push(null); // no data
args.push(function(data) {
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
result[key] = data[key];
}
});
};
for (var i = 0; i < keys.length; i += 10) {
makeBatch(keys.slice(i, i + 10));
}
args.push(function() {
onDone(result);
});
Ajax.chainGetJSON.apply(null, args);
};
SchemaAlignment._cleanName = function(s) {
return s.replace(/\W/g, " ").replace(/\s+/g, " ").toLowerCase();
};
SchemaAlignment.createNewRootNode = function() {
var rootNode = null;
var links = [];
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
var target = {
nodeType: "cell-as-topic",
columnName: column.name,
createForNoReconMatch: true
};
if ((column.reconConfig) &&
ReconciliationManager.isFreebaseIdOrMid(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace) &&
(column.reconConfig.type)) {
target.type = {
id: column.reconConfig.type.id,
name: column.reconConfig.type.name
};
}
if (column.name == theProject.columnModel.keyColumnName) {
rootNode = target;
} else {
links.push({
property: null,
target: target
});
}
}
rootNode = rootNode || { nodeType: "cell-as-topic" };
rootNode.links = links;
return rootNode;
};
function SchemaAlignmentDialog(protograph, onDone) {
this._onDone = onDone;
this._hasUnsavedChanges = false;
this._createDialog();
this._reset(protograph, true);
}
SchemaAlignmentDialog.prototype._reset = function(protograph, initial) {
this._originalProtograph = protograph || { rootNodes: [] };
this._protograph = cloneDeep(this._originalProtograph); // this is what can be munched on
if (!this._protograph.rootNodes.length) {
this._protograph.rootNodes.push(SchemaAlignment.createNewRootNode());
}
$(this._nodeTable).empty();
this._nodeUIs = [];
for (var i = 0; i < this._protograph.rootNodes.length; i++) {
this._nodeUIs.push(new SchemaAlignmentDialog.UINode(
this,
this._protograph.rootNodes[i],
this._nodeTable,
{
expanded: true,
mustBeCellTopic: true
}
));
}
this.preview(initial);
};
SchemaAlignmentDialog.prototype._save = function(onDone) {
var self = this;
var protograph = this.getJSON();
Refine.postProcess(
"freebase",
"save-protograph",
{},
{ protograph: JSON.stringify(protograph) },
{},
{
onDone: function() {
theProject.overlayModels.freebaseProtograph = protograph;
self._elmts.statusIndicator.hide();
self._hasUnsavedChanges = false;
if (onDone) onDone();
}
}
);
};
SchemaAlignmentDialog.prototype._createDialog = function() {
var self = this;
var frame = $(DOM.loadHTML("freebase", "scripts/dialogs/schema-alignment/schema-alignment-dialog.html"));
var elmts = this._elmts = DOM.bind(frame);
this._level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(self._level - 1);
};
elmts.dialogHeader.text($.i18n._('fb-interface')["dialog-header"]);
elmts.body_text.text($.i18n._('fb-interface')["body-text"]);
elmts.find_more.text($.i18n._('fb-interface')["find-more"]);
elmts.skeleton.text($.i18n._('fb-interface')["skeleton"]);
elmts.skeleton.text($.i18n._('fb-interface')["mql-preview"]);
elmts.skeleton.text($.i18n._('fb-interface')["tripleloader-preview"]);
elmts.saveButton.text($.i18n._('fb-buttons')["save"]);
elmts.saveAndLoadButton.text($.i18n._('fb-buttons')["save-load"]);
elmts.closeButton.text($.i18n._('fb-buttons')["close"]);
elmts.resetButton.text($.i18n._('fb-buttons')["reset"]);
elmts.saveButton.click(function() {
self._save();
});
elmts.saveAndLoadButton.click(function() {
self._save(function() {
dismiss();
FreebaseExtension.handlers.loadIntoFreebase();
});
});
elmts.resetButton.click(function() {
self._reset(null);
});
elmts.closeButton.click(function() {
if (!self._hasUnsavedChanges || window.confirm($.i18n._('fb-schema-alignment')["close-confirm"])) {
dismiss();
}
});
$("#schema-alignment-tabs").tabs();
$("#schema-alignment-tabs-preview-mqllike").css("display", "");
$("#schema-alignment-tabs-preview-tripleloader").css("display", "");
this._previewPanes = $(".schema-alignment-dialog-preview");
this._canvas = $(".schema-alignment-dialog-canvas");
this._nodeTable = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._canvas)[0];
};
SchemaAlignmentDialog.prototype.getJSON = function() {
var rootNodes = [];
for (var i = 0; i < this._nodeUIs.length; i++) {
var node = this._nodeUIs[i].getJSON();
if (node !== null) {
rootNodes.push(node);
}
}
return {
rootNodes: rootNodes
};
};
SchemaAlignmentDialog.prototype.preview = function(initial) {
var self = this;
this._previewPanes.empty();
if (!(initial)) {
this._elmts.statusIndicator.show().text($.i18n._('fb-schema-alignment')["status-warning"]);
this._hasUnsavedChanges = true;
}
var protograph = this.getJSON();
$.post(
"command/freebase/preview-protograph?" + $.param({ project: theProject.id }),
{ protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
if ("mqllike" in data) {
$(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2));
}
if ("tripleloader" in data) {
$(self._previewPanes[1]).text(data.tripleloader);
}
},
"json"
);
};
SchemaAlignmentDialog._findColumn = function(cellIndex) {
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
if (column.cellIndex == cellIndex) {
return column;
}
}
return null;
};

39
extensions/freebase/module/scripts/dialogs/schema-alignment/schema-alignment-dialog.html
View File

@ -1,39 +0,0 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<p class="body-text" bind="body_text">
<a href="https://github.com/OpenRefine/OpenRefine/wiki/Schema-Alignment" target="_blank" bind="find_more"></a>
</p>
<div id="schema-alignment-tabs" class="refine-tabs">
<ul>
<li><a href="#schema-alignment-tabs-protograph" bind="skeleton"></a></li>
<li><a href="#schema-alignment-tabs-preview-mqllike" bind="mql_preview"></a></li>
<li><a href="#schema-alignment-tabs-preview-tripleloader" bind="tripleloader_preview"></a></li>
</ul>
<div id="schema-alignment-tabs-protograph">
<div class="schema-alignment-dialog-canvas"></div>
</div>
<div id="schema-alignment-tabs-preview-mqllike" style="display: none;">
<div class="schema-alignment-dialog-preview"></div>
</div>
<div id="schema-alignment-tabs-preview-tripleloader" style="display: none;">
<div class="schema-alignment-dialog-preview"></div>
</div>
</div>
</div>
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-normal layout-full"><table><tr>
<td>
<button class="button" bind="resetButton"></button>
</td>
<td style="text-align:right;">
<button class="button button-primary" bind="saveButton"></button>
<button class="button" bind="saveAndLoadButton"></button>
</td>
<td style="text-align:center;" width="30%">
<span class="schema-alignment-status-indicator" bind="statusIndicator"></span>
</td>
<td style="text-align:right;" width="1%">
<button class="button" bind="closeButton"></button>
</td>
</tr></table></div></div>
</div>

381
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js
View File

@ -1,381 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
SchemaAlignmentDialog.UILink = function(dialog, link, table, options, parentUINode) {
this._dialog = dialog;
this._link = link;
this._options = options;
this._parentUINode = parentUINode;
// Make sure target node is there
this._link.target = this._link.target || { nodeType: "cell-as-value" }
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-link-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-link-toggle").attr("width", "1%").addClass("padded");
$(this._tdDetails).addClass("schema-alignment-link-details").attr("width", "90%");
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
var self = this;
var show = function() {
if (self._options.expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._options.expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._options.expanded = !self._options.expanded;
$(this).attr("src", self._options.expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
this._renderMain();
this._renderDetails();
};
SchemaAlignmentDialog.UILink.prototype._renderMain = function() {
$(this._tdMain).empty();
var label = this._link.property !== null ?
(this._link.property.id + ((this._link.condition) ? " [?]" : "")) :
"property?";
var self = this;
$('<img />')
.attr("title", "remove property")
.attr("src", "images/close.png")
.css("cursor", "pointer")
.prependTo(this._tdMain)
.click(function() {
window.setTimeout(function() {
self._parentUINode.removeLink(self);
self._tr.parentNode.removeChild(self._tr);
self._dialog.preview();
}, 100);
});
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-link-tag")
.html(label)
.appendTo(this._tdMain)
.click(function(evt) {
self._startEditProperty(this);
});
$('<img />').attr("src", "images/arrow-start.png").prependTo(a);
$('<img />').attr("src", "images/arrow-end.png").appendTo(a);
};
SchemaAlignmentDialog.UILink.prototype._renderDetails = function() {
if (this._targetUI) {
this._targetUI.dispose();
}
if (this._tableDetails) {
this._tableDetails.remove();
}
this._tableDetails = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv);
this._targetUI = new SchemaAlignmentDialog.UINode(
this._dialog,
this._link.target,
this._tableDetails[0],
{ expanded: "links" in this._link.target && this._link.target.links.length > 0 });
};
SchemaAlignmentDialog.UILink.prototype._startEditProperty = function(elmt) {
var sourceTypeID = this._parentUINode.getExpectedType();
var targetNode = this._targetUI._node;
var targetTypeID = "type" in targetNode && targetNode.type !== null ? targetNode.type.id : null;
var targetTypeName = "columnNames" in targetNode ? targetNode.columnNames[0] : null;
if (sourceTypeID !== null) {
var self = this;
var dismissBusy = DialogSystem.showBusy();
var instanceCount = 0;
var outgoing = [];
var incoming = [];
var onDone = function(properties) {
dismissBusy();
self._showPropertySuggestPopup(
elmt,
properties
);
};
SchemaAlignmentDialog.UILink._getPropertiesOfType(
sourceTypeID,
targetTypeID,
targetTypeName,
onDone
);
} else {
this._showPropertySuggestPopup(elmt, []);
}
};
SchemaAlignmentDialog.UILink._getPropertiesOfType = function(typeID, targetTypeID, targetTypeName, onDone) {
var done = false;
var params = {
"type" : typeID
};
if (targetTypeID !== null) {
params.expects = targetTypeID;
} else if (targetTypeName !== null) {
params.expects = targetTypeName;
}
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?" + $.param(params) + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
onDone(data.properties || []);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
SchemaAlignmentDialog.UILink.prototype._showPropertySuggestPopup = function(elmt, suggestions) {
self = this;
var menu = MenuSystem.createMenu().width("350px");
var commitProperty = function(p) {
window.setTimeout(function() { MenuSystem.dismissAll(); }, 100);
if ("id2" in p) {
// self._targetUI.dispose();
self._link.property = {
id: p.id,
name: p.name
};
self._link.target = {
nodeType: "anonymous",
links: [{
property: {
id: p.id2,
name: p.name2
},
target: self._link.target
}]
};
self._renderDetails();
} else {
self._link.property = {
id: p.id,
name: p.name
};
}
var conditionColumnName = conditionalSelect[0].value;
if (conditionColumnName != "") {
self._link.condition = { columnName: conditionColumnName };
} else {
delete self._link.condition;
}
self._configureTarget();
};
var divConditional = $('<div>')
.addClass("schema-alignment-link-menu-section")
.html($.i18n._('fb-schema-alignment')["assert-link-found"]+"<br/>").appendTo(menu);
var conditionalSelect = $('<select>').appendTo(divConditional);
$('<option>')
.text("(always assert)")
.attr("value", "")
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
for (var c = 0; c < theProject.columnModel.columns.length; c++) {
var column = theProject.columnModel.columns[c];
var option = $('<option>')
.text(column.name)
.attr("value", column.name)
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
if ((self._link.condition) && column.name == self._link.condition.columnName) {
option.attr("selected", "true");
}
}
var divSearch;
if (suggestions.length > 0) {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section")
.css("margin-bottom", "2em")
.html('<div>'+$.i18n._('fb-schema-alignment')["search-pick-property"]+'</div>').appendTo(menu);
var createSuggestion = function(suggestion) {
var menuItem = MenuSystem.createMenuItem().appendTo(menu);
$('<span>')
.text(suggestion.name)
.attr("title", suggestion.id)
.appendTo(menuItem);
if ("name2" in suggestion) {
$('<span>').html(" &raquo; ").appendTo(menuItem);
$('<span>')
.text(suggestion.name2)
.attr("title", suggestion.id2)
.appendTo(menuItem);
}
menuItem.click(function() {
commitProperty(suggestion);
});
};
for (var i = 0; i < suggestions.length && i < 10; i++) {
createSuggestion(suggestions[i]);
}
} else {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section-last")
.html('<div>'+$.i18n._('fb-schema-alignment')["search-property"]+'</div>').appendTo(menu);
}
var input = $('<input />').appendTo($('<div>').appendTo(divSearch));
MenuSystem.showMenu(menu, function(){});
MenuSystem.positionMenuAboveBelow(menu, $(elmt));
var suggestOptions = {
filter : '(all type:/type/property)'
};
var sourceTypeID = this._parentUINode.getExpectedType();
if (sourceTypeID !== null) {
suggestOptions.filter = '(all type:/type/property (should namespace:' + sourceTypeID + '))'
}
input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
input[0].focus();
};
SchemaAlignmentDialog.UILink.prototype.getJSON = function() {
if ("property" in this._link && this._link.property !== null &&
"target" in this._link && this._link.target !== null) {
var targetJSON = this._targetUI.getJSON();
if (targetJSON !== null) {
var json = {
property: cloneDeep(this._link.property),
target: targetJSON
};
if (this._link.condition) {
json.condition = cloneDeep(this._link.condition);
}
return json;
}
}
return null;
};
SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
var self = this;
var dismissBusy = DialogSystem.showBusy();
$.getJSON(
"https://www.googleapis.com/freebase/v1/mqlread?key="+ Freebase.API_KEY + "&query=" + JSON.stringify({
"id" : this._link.property.id,
"type" : "/type/property",
"expected_type" : {
"id" : null,
"name" : null,
"/freebase/type_hints/mediator" : null
}
}) + "&callback=?",
null,
function(o) {
dismissBusy();
if ("result" in o) {
var expected_type = o.result.expected_type;
self._link.target.type = {
id: expected_type.id,
name: expected_type.name
};
if (expected_type["/freebase/type_hints/mediator"] === true) {
self._link.target.nodeType = "anonymous";
} else if (expected_type.id == "/type/key") {
self._link.target.nodeType = "cell-as-key";
} else if (expected_type.id.match(/^\/type\//)) {
self._link.target.nodeType = "cell-as-value";
} else if (!("topic" in self._link.target)) {
self._link.target.nodeType = "cell-as-topic";
self._link.target.createForNoReconMatch = true;
}
self._targetUI.render();
}
self._renderMain();
self._dialog.preview();
},
"jsonp"
);
};

754
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
View File

@ -1,754 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
SchemaAlignmentDialog.UINode = function(dialog, node, table, options) {
this._dialog = dialog;
this._node = node;
this._options = options;
if ("columnName" in this._node) {
this._node.columnNames = [ this._node.columnName ];
delete this._node.columnName;
}
this._linkUIs = [];
this._detailsRendered = false;
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide();
$(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide();
this._renderMain();
this._expanded = options.expanded;
if (this._isExpandable()) {
this._showExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.dispose = function() {
// nothing for now
};
SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) {
for (var i = this._linkUIs.length - 1; i >= 0; i--) {
if (this._linkUIs[i] === linkUI) {
this._linkUIs.splice(i, 1);
this._node.links.splice(i, 1);
break;
}
}
};
SchemaAlignmentDialog.UINode.prototype._isExpandable = function() {
return this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "anonymous" ||
this._node.nodeType == "topic";
};
SchemaAlignmentDialog.UINode.prototype.render = function() {
this._renderMain();
if (this._isExpandable()) {
this._showExpandable();
} else {
this._hideExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() {
if ("type" in this._node) {
return this._node.type.id;
}
return null;
};
SchemaAlignmentDialog.UINode.prototype._renderMain = function() {
$(this._tdMain).empty();
var self = this;
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-node-tag")
.appendTo(this._tdMain)
.click(function(evt) {
self._showNodeConfigDialog();
});
if (this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "cell-as-value" ||
this._node.nodeType == "cell-as-key") {
if ("columnNames" in this._node) {
for (var c = 0; c < this._node.columnNames.length; c++) {
if (c > 0) {
$('<span>').text(", ").appendTo(a);
}
$('<span>')
.text(this._node.columnNames[c])
.addClass("schema-alignment-node-column")
.appendTo(a);
}
$('<span>').text(
this._node.columnNames.length > 1 ? " "
+ $.i18n._('fb-schema-alignment')["cells"] : " "
+ $.i18n._('fb-schema-alignment')["cell"]).appendTo(a);
} else {
a.html(this._options.mustBeCellTopic ? $.i18n._('fb-schema-alignment')["which-column"] : $.i18n._('fb-schema-alignment')["configure"]);
}
} else if (this._node.nodeType == "topic") {
if ("topic" in this._node) {
a.html(this._node.topic.name);
} else if ("id" in this._node) {
a.html(this._node.topic.id);
} else {
a.html($.i18n._('fb-schema-alignment')["which-topic"]);
}
} else if (this._node.nodeType == "value") {
if ("value" in this._node) {
a.html(this._node.value);
} else {
a.html($.i18n._('fb-schema-alignment')["what-value"]);
}
} else if (this._node.nodeType == "anonymous") {
a.html("("+$.i18n._('fb-schema-alignment')["anonymous"]+")");
}
};
SchemaAlignmentDialog.UINode.prototype._showExpandable = function() {
$(this._tdToggle).show();
$(this._tdDetails).show();
if (this._detailsRendered) {
return;
}
this._detailsRendered = true;
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
this._renderDetails();
var self = this;
var show = function() {
if (self._expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._expanded = !self._expanded;
$(this).attr("src", self._expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
};
SchemaAlignmentDialog.UINode.prototype._hideExpandable = function() {
$(this._tdToggle).hide();
$(this._tdDetails).hide();
};
SchemaAlignmentDialog.UINode.prototype._renderDetails = function() {
var self = this;
this._tableLinks = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv)[0];
if ("links" in this._node && this._node.links !== null) {
for (var i = 0; i < this._node.links.length; i++) {
this._linkUIs.push(new SchemaAlignmentDialog.UILink(
this._dialog,
this._node.links[i],
this._tableLinks,
{ expanded: true },
this
));
}
}
var divFooter = $('<div></div>').addClass("padded").appendTo(this._expandedDetailDiv);
$('<a href="javascript:{}"></a>')
.addClass("action")
.text($.i18n._('fb-schema-alignment')["add-property"])
.appendTo(divFooter)
.click(function() {
var newLink = {
property: null,
target: {
nodeType: "cell-as-value"
}
};
self._linkUIs.push(new SchemaAlignmentDialog.UILink(
self._dialog,
newLink,
self._tableLinks,
{
expanded: true,
mustBeCellTopic: false
},
self
));
});
};
SchemaAlignmentDialog.UINode.prototype._showColumnPopupMenu = function(elmt) {
self = this;
var menu = [];
if (!this._options.mustBeCellTopic) {
menu.push({
label: $.i18n._('fb-schema-alignment')["anonymous-node"],
click: function() {
self._node.nodeType = "anonymous";
self._showExpandable();
self._renderMain();
}
});
menu.push({
label: $.i18n._('fb-schema-alignment')["freebase-topic"],
click: function() {
self._node.nodeType = "topic";
self._hideExpandable();
self._renderMain();
}
});
menu.push({
label: $.i18n._('fb-schema-alignment')["value"],
click: function() {
self._node.nodeType = "value";
self._hideExpandable();
self._renderMain();
}
});
menu.push({}); // separator
}
var columns = theProject.columnModel.columns;
var createColumnMenuItem = function(index) {
menu.push({
label: columns[index].name,
click: function() {
self._node.nodeType = "cell-as-topic";
self._node.columnNames = [ columns[index].name ];
self._showExpandable();
self._renderMain();
}
});
};
for (var i = 0; i < columns.length; i++) {
createColumnMenuItem(i);
}
MenuSystem.createAndShowStandardMenu(menu, elmt);
};
SchemaAlignmentDialog.UINode.prototype._showNodeConfigDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("750px");
var header = $('<div></div>').addClass("dialog-header").text($.i18n._('fb-schema-alignment')["skeleton-node"]).appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
/*--------------------------------------------------
* Body
*--------------------------------------------------
*/
var literalTypeSelectHtml =
'<option value="/type/text" checked>'+$.i18n._('fb-schema-alignment')["text"]+'</option>' +
'<option value="/type/int">'+$.i18n._('fb-schema-alignment')["int"]+'</option>' +
'<option value="/type/float">'+$.i18n._('fb-schema-alignment')["float"]+'</option>' +
'<option value="/type/double">'+$.i18n._('fb-schema-alignment')["double"]+'</option>' +
'<option value="/type/boolean">'+$.i18n._('fb-schema-alignment')["boolean"]+'</option>' +
'<option value="/type/datetime">'+$.i18n._('fb-schema-alignment')["date-time"]+'</option>' +
'<option value="/type/rawstring">'+$.i18n._('fb-schema-alignment')["rawstring"]+'</option>';
var html = $(
'<div class="grid-layout layout-looser layout-full"><table>' +
'<tr>' +
'<td>' +
'<div class="grid-layout layout-loose"><table>' +
'<tr>' +
'<td>' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="cell-as" bind="radioNodeTypeCellAs" /> '+$.i18n._('fb-schema-alignment')["set-to-cell"]+'' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td>' +
'<div class="grid-layout layout-normal"><table>' +
'<tr>' +
'<td><div class="schema-alignment-node-dialog-column-list grid-layout layout-tightest" bind="divColumns"></div></td>' +
'<td>' +
'<div class="grid-layout layout-tight"><table cols="4">' +
'<tr>' +
'<td colspan="4">'+$.i18n._('fb-schema-alignment')["cell-content-used"]+'</td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-topic" bind="radioNodeTypeCellAsTopic" /></td>' +
'<td colspan="3">'+$.i18n._('fb-schema-alignment')["specify-fb-topic"]+'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="3">'+$.i18n._('fb-schema-alignment')["type-new-topic"]+'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="3"><input bind="cellAsTopicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-value" bind="radioNodeTypeCellAsValue" /></td>' +
'<td colspan="3">'+$.i18n._('fb-schema-alignment')["literal-value"]+'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">'+$.i18n._('fb-schema-alignment')["literal-type"]+'</td>' +
'<td colspan="1"><select bind="cellAsValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">'+$.i18n._('fb-schema-alignment')["text-language"]+'</td>' +
'<td colspan="1"><input bind="cellAsValueLanguageInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-key" bind="radioNodeTypeCellAsKey" /></td>' +
'<td colspan="3">'+$.i18n._('fb-schema-alignment')["key-namespace"]+'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">'+$.i18n._('fb-schema-alignment')["namespace"]+'</td>' +
'<td colspan="1"><input bind="cellAsKeyInput" /></td>' +
'</tr>' +
'</table></div>' +
'</td>' +
'</tr>' +
'</table></div>' +
'</td>' +
'</tr>' +
'</table></div>' +
'</td>' +
'<td>' +
'<div class="grid-layout layout-normal"><table>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="anonymous" bind="radioNodeTypeAnonymous" /> '+$.i18n._('fb-schema-alignment')["generate-anonymous"]+'' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>'+$.i18n._('fb-schema-alignment')["assign-type"]+'</td>' +
'<td>&nbsp;<input bind="anonymousNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="topic" bind="radioNodeTypeTopic" /> '+$.i18n._('fb-schema-alignment')["use-existing-topic"]+'' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Topic</td>' +
'<td><input bind="topicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="value" bind="radioNodeTypeValue" /> '+$.i18n._('fb-schema-alignment')["use-literal-value"]+
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>'+$.i18n._('fb-schema-alignment')["value"]+'</td>' +
'<td><input bind="valueNodeTypeValueInput" /></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>'+$.i18n._('fb-schema-alignment')["value-type"]+'</td>' +
'<td><select bind="valueNodeTypeValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>'+$.i18n._('fb-schema-alignment')["language"]+'</td>' +
'<td><input bind="valueNodeTypeLanguageInput" /></td>' +
'</tr>' +
'</table></div>' +
'</td>' +
'</tr>' +
'</table></div>'
).appendTo(body);
var elmts = DOM.bind(html);
var tableColumns = $('<table></table>')
.attr("cellspacing", "5")
.attr("cellpadding", "0")
.appendTo(elmts.divColumns)[0];
var columnMap = {};
if ("columnNames" in self._node) {
for (var i = 0; i < self._node.columnNames.length; i++) {
columnMap[self._node.columnNames[i]] = true;
}
}
var makeColumnChoice = function(column, columnIndex) {
var tr = tableColumns.insertRow(tableColumns.rows.length);
var radio = $('<input />')
.attr("type", "checkbox")
.attr("value", column.name)
.attr("name", "schema-align-node-dialog-column")
.appendTo(tr.insertCell(0))
.click(function() {
elmts.radioNodeTypeCellAs[0].checked = true;
if ("reconConfig" in column) {
var typeID = column.reconConfig.type.id;
var typeName = column.reconConfig.type.name;
elmts.cellAsTopicNodeTypeInput[0].value = typeName;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", { "id" : typeID, "name" : typeName });
elmts.radioNodeTypeCellAsTopic[0].checked = true;
}
});
if (column.name in columnMap) {
radio.prop("checked", true);
}
$('<span></span>').text(column.name).appendTo(tr.insertCell(1));
};
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
makeColumnChoice(columns[i], i);
}
elmts.anonymousNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeAnonymous[0].checked = true; })
.suggestT({ filter : "(all type:/type/type)" });
elmts.topicNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeTopic[0].checked = true; })
.suggest({});
elmts.valueNodeTypeValueInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeValueTypeSelect
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeLanguageInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; })
.suggest({ filter : "(all type:/type/lang)" });
elmts.cellAsTopicNodeTypeInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
})
.suggestT({ filter : "(all type:/type/type)" });
elmts.cellAsValueTypeSelect
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
});
elmts.cellAsValueLanguageInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
})
.suggest({ filter : "(all type:/type/lang)" });
elmts.cellAsKeyInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsKey[0].checked = true;
})
.suggest({ filter : "(all type:/type/namespace)" });
elmts.radioNodeTypeCellAsTopic[0].checked = true; // just make sure some subtype is selected
if (this._node.nodeType.match(/^cell-as-/)) {
elmts.radioNodeTypeCellAs[0].checked = true;
if (this._node.nodeType == "cell-as-topic") {
elmts.radioNodeTypeCellAsTopic[0].checked = true;
} else if (this._node.nodeType == "cell-as-value") {
elmts.radioNodeTypeCellAsValue[0].checked = true;
} else if (this._node.nodeType == "cell-as-key") {
elmts.radioNodeTypeCellAsKey[0].checked = true;
}
} else if (this._node.nodeType == "anonymous") {
elmts.radioNodeTypeAnonymous[0].checked = true;
} else if (this._node.nodeType == "topic") {
elmts.radioNodeTypeTopic[0].checked = true;
} else if (this._node.nodeType == "value") {
elmts.radioNodeTypeValue[0].checked = true;
}
if ("type" in this._node) {
elmts.anonymousNodeTypeInput[0].value = this._node.type.name;
elmts.anonymousNodeTypeInput.data("data.suggest", this._node.type);
elmts.cellAsTopicNodeTypeInput[0].value = this._node.type.name;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", this._node.type);
}
if ("topic" in this._node) {
elmts.topicNodeTypeInput[0].value = this._node.topic.name;
elmts.topicNodeTypeInput.data("data.suggest", this._node.topic);
}
if ("namespace" in this._node) {
elmts.cellAsKeyInput[0].value = this._node.namespace.name;
elmts.cellAsKeyInput.data("data.suggest", this._node.namespace);
}
if ("lang" in this._node) {
elmts.valueNodeTypeLanguageInput[0].value = this._node.lang;
elmts.valueNodeTypeLanguageInput.data("data.suggest", { id: this._node.lang });
elmts.cellAsValueLanguageInput[0].value = this._node.lang;
elmts.cellAsValueLanguageInput.data("data.suggest", { id: this._node.lang });
}
if ("valueType" in this._node) {
elmts.valueNodeTypeValueTypeSelect[0].value = this._node.valueType;
elmts.cellAsValueTypeSelect[0].value = this._node.valueType;
}
/*--------------------------------------------------
* Footer
*--------------------------------------------------
*/
var getResultJSON = function() {
var node = {
nodeType: $("input[name='schema-align-node-dialog-node-type']:checked")[0].value
};
if (node.nodeType == "cell-as") {
node.nodeType = $("input[name='schema-align-node-dialog-node-subtype']:checked")[0].value;
node.columnNames = $("input[name='schema-align-node-dialog-column']:checked").map(function() {
return this.getAttribute("value");
}).get();
if (node.columnNames.length == 0) {
alert($.i18n._('fb-schema-alignment')["column-warning"]);
return null;
}
if (node.nodeType == "cell-as-topic") {
var t = elmts.cellAsTopicNodeTypeInput.data("data.suggest");
if (!(t)) {
alert($.i18n._('fb-schema-alignment')["new-node-warning"]);
elmts.cellAsTopicNodeTypeInput.focus();
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "cell-as-value") {
node.valueType = elmts.cellAsValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.cellAsValueLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
} else if (node.nodeType == "cell-as-key") {
var t = elmts.cellAsKeyInput.data("data.suggest");
if (!(t)) {
alert($.i18n._('fb-schema-alignment')["namespace-warning"]);
return null;
}
node.namespace = {
id: t.id,
name: t.name
};
}
} else if (node.nodeType == "anonymous") {
var t = elmts.anonymousNodeTypeInput.data("data.suggest");
if (!(t)) {
alert($.i18n._('fb-schema-alignment')["anonymous-node-warning"]);
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "topic") {
var t = elmts.topicNodeTypeInput.data("data.suggest");
if (!(t)) {
alert($.i18n._('fb-schema-alignment')["specify-topic-warning"]);
return null;
}
node.topic = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "value") {
node.value = $.trim(elmts.valueNodeTypeValueInput[0].value);
if (!node.value.length) {
alert($.i18n._('fb-schema-alignment')["specify-value-warning"]);
return null;
}
node.valueType = elmts.valueNodeTypeValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.valueNodeTypeLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
}
return node;
};
$('<button class="button"></button>').html("&nbsp;&nbsp;OK&nbsp;&nbsp;").click(function() {
var node = getResultJSON();
if (node !== null) {
DialogSystem.dismissUntil(level - 1);
self._node = node;
self.render();
self._dialog.preview();
}
}).appendTo(footer);
$('<button class="button"></button>').text($.i18n._('fb-buttons')["cancel"]).click(function() {
DialogSystem.dismissUntil(level - 1);
}).appendTo(footer);
var level = DialogSystem.showDialog(frame);
};
SchemaAlignmentDialog.UINode.prototype.getJSON = function() {
var result = null;
var getLinks = false;
if (this._node.nodeType.match(/^cell-as-/)) {
if (!("columnNames" in this._node) || !this._node.columnNames) {
return null;
}
if (this._node.nodeType == "cell-as-topic") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
type: "type" in this._node ? cloneDeep(this._node.type) : { "id" : "/common/topic", "name" : "Topic", "cvt" : false }
};
getLinks = true;
} else if (this._node.nodeType == "cell-as-value") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "cell-as-key") {
if (!("namespace" in this._node) || !this._node.namespace) {
return null;
}
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
namespace: cloneDeep(this._node.namespace)
};
}
} else if (this._node.nodeType == "topic") {
if (!("topic" in this._node) || !this._node.topic) {
return null;
}
result = {
nodeType: this._node.nodeType,
topic: cloneDeep(this._node.topic)
};
getLinks = true;
} else if (this._node.nodeType == "value") {
if (!("value" in this._node) || !this._node.value) {
return null;
}
result = {
nodeType: this._node.nodeType,
value: this._node.value,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "anonymous") {
if (!("type" in this._node) || !this._node.type) {
return null;
}
result = {
nodeType: this._node.nodeType,
type: cloneDeep(this._node.type)
};
getLinks = true;
}
if (!result) {
return null;
}
if (getLinks) {
var links = [];
for (var i = 0; i < this._linkUIs.length; i++) {
var link = this._linkUIs[i].getJSON();
if (link !== null) {
links.push(link);
}
}
result.links = links;
}
return result;
};

192
extensions/freebase/module/scripts/extension.js
View File

@ -1,192 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
var FreebaseExtension = { handlers: {} };
// Internationalization init
var lang = navigator.language.split("-")[0]
|| navigator.userLanguage.split("-")[0];
var dictionary = "";
$.ajax({
url : "command/core/load-language?",
type : "POST",
async : false,
data : {
module : "freebase",
// lang : lang
},
success : function(data) {
dictionary = data;
}
});
$.i18n.setDictionary(dictionary);
// End internationalization
FreebaseExtension.handlers.setFreebaseApiKey = function() {
var value = window.prompt("Set Freebase API Key:");
if (value !== null) {
$.post(
"command/core/set-preference",
{
name : "freebase.api.key",
value : JSON.stringify(value)
},
function(o) {
if (o.code == "error") {
alert(o.message);
}
},
"json"
);
CustomSuggest.setFreebaseAPIKey(value);
}
};
FreebaseExtension.handlers.editSchemaAlignment = function() {
new SchemaAlignmentDialog(theProject.overlayModels.freebaseProtograph, function(newProtograph) {});
};
FreebaseExtension.handlers.loadIntoFreebase = function() {
new FreebaseLoadingDialog();
};
FreebaseExtension.handlers.browseToDataLoad = function() {
// The form has to be created as part of the click handler. If you create it
// inside the getJSON success handler, it won't work.
var form = document.createElement("form");
$(form)
.css("display", "none")
.attr("method", "GET")
.attr("target", "dataload");
document.body.appendChild(form);
var w = window.open("about:blank", "dataload");
$.getJSON(
"command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobID" }),
null,
function(data) {
if (data.value == null) {
alert($.i18n._('fb-menu')["warning-load"]);
} else {
$(form).attr("action", "http://refinery.freebaseapps.com/load/" + data.value);
form.submit();
w.focus();
}
document.body.removeChild(form);
}
);
};
FreebaseExtension.handlers.importQAData = function() {
Refine.postProcess(
"freebase-extension",
"import-qa-data",
{},
{},
{ cellsChanged: true }
);
};
ExtensionBar.addExtensionMenu({
"id" : "freebase",
"label" : $.i18n._('fb-menu')["freebase"],
"submenu" : [
{
"id" : "freebase/set-api-key",
label: $.i18n._('fb-menu')["set-api-key"],
click: FreebaseExtension.handlers.setFreebaseApiKey
},
{
"id" : "freebase/schema-alignment",
label: $.i18n._('fb-menu')["align-schema"],
click: function() { FreebaseExtension.handlers.editSchemaAlignment(false); }
},
{
"id" : "freebase/load-info-freebase",
label: $.i18n._('fb-menu')["load"],
click: function() { FreebaseExtension.handlers.loadIntoFreebase(); }
},
{},
{
"id" : "freebase/browse-load",
label: $.i18n._('fb-menu')["browse-data-load"],
click: function() { FreebaseExtension.handlers.browseToDataLoad(); }
},
{
"id" : "freebase/import-qa-data",
label: $.i18n._('fb-menu')["import-qa"],
click: function() { FreebaseExtension.handlers.importQAData(); }
}
]
});
DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
var columnIndex = Refine.columnNameToColumnIndex(column.name);
var doAddColumnFromFreebase = function() {
var o = DataTableView.sampleVisibleRows(column);
new ExtendDataPreviewDialog(
column,
columnIndex,
o.rowIndices,
function(extension) {
Refine.postProcess(
"freebase",
"extend-data",
{
baseColumnName: column.name,
columnInsertIndex: columnIndex + 1
},
{
extension: JSON.stringify(extension)
},
{ rowsChanged: true, modelsChanged: true }
);
}
);
};
MenuSystem.insertAfter(
menu,
[ "core/edit-column", "core/add-column-by-fetching-urls" ],
{
id: "freebase/add-columns-from-freebase",
label: $.i18n._('fb-menu')["add-columns"],
click: doAddColumnFromFreebase
}
);
});

63
extensions/freebase/module/scripts/util/freebase.js
View File

@ -1,63 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
var Freebase = {};
Freebase.API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
Freebase.mqlread = function(query, options, onDone) {
var params = {};
// TODO: Options need to be handled differently for new API - but this doesn't appear to be used
if (options) {
for (var n in options) {
if (options.hasOwnProperty(n)) {
var v = options[n];
if (typeof v != "string") {
v = JSON.stringify(v);
}
queryEnv[n] = v;
}
}
}
params.query = JSON.stringify(query);
$.getJSON(
"https://www.googleapis.com/freebase/v1/mqlread?key=" + Freebase.API_KEY + $.param(params) + "&callback=?",
null,
onDone,
"jsonp"
);
};

71
extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less
View File

@ -1,71 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.extend-data-preview-dialog .suggested-property-container {
border: 1px solid #aaa;
padding: 5px;
overflow: auto;
height: 375px;
}
.extend-data-preview-dialog .suggested-property {
padding: 5px;
}
.extend-data-preview-dialog input.property-suggest {
display: block;
padding: 2%;
width: 96%;
}
.extend-data-preview-dialog .preview-container {
border: 1px solid #aaa;
overflow: auto;
}
.extend-data-preview-dialog .preview-container table {
border-collapse: collapse;
}
.extend-data-preview-dialog .preview-container td, .extend-data-preview-dialog .preview-container th {
padding: 3px 5px;
border-bottom: 1px solid #ddd;
border-right: 1px solid #ddd;
}
.extend-data-preview-dialog .preview-container th img {
vertical-align: top;
margin-left: 5px;
}

92
extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less
View File

@ -1,92 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.freebase-loading-dialog-functional-tab-panel {
height: 400px !important;
}
.freebase-loading-tripleloader-data {
width: 99%;
overflow: scroll;
border: 1px solid #aaa;
white-space: pre;
padding: 0.3em 0.5em 0.5em 0.5em;
font-family: monospace;
}
.freebase-loading-tripleloader-message {
height: 400px;
overflow: auto;
border: 1px solid #aaa;
padding: 1em;
}
.freebase-loading-tripleloader-message h2 {
font-size: 150%;
font-weight: normal;
}
.freebase-loading-tripleloader-message h2 span {
font-size; 130%;
font-weight: bold;
}
.freebase-loading-tripleloader-message h4 {
font-size: 120%;
margin-top: 2em;
font-weight: normal;
}
.freebase-loading-tripleloader-message a {
font-size: 120%;
font-weight: bold;
}
.freebase-loading-authorization {
margin: 0em 2em;
padding: 0.6em 0.8em;
border: 1px solid #ccc;
background-color: #fff;
-moz-border-radius: 0.8em;
-webkit-border-radius: 0.8em;
}
.freebase-loading-tripleloader-info {
margin-bottom: 0.5em;
}
.freebase-loading-tripleloader-info textarea {
width: 99%;
height: 5em;
}

149
extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less
View File

@ -1,149 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.schema-alignment-dialog-canvas {
height: 375px;
overflow: auto;
padding: 10px;
margin-top: 3px;
background: white;
}
table.schema-alignment-table-layout {
border-collapse: collapse;
margin: 0px;
padding: 0px;
margin-bottom: 1em;
}
table.schema-alignment-table-layout .padded {
padding: 3px 6px;
}
div.schema-alignment-detail-container {
border-left: 3px solid #eee;
}
td.schema-alignment-node-main, td.schema-alignment-link-main {
white-space: pre;
width: 300px;
}
td.schema-alignment-node-toggle, td.schema-alignment-link-toggle {
white-space: pre;
width: 1%;
}
td.schema-alignment-node-toggle img, td.schema-alignment-link-toggle img {
cursor: pointer;
vertical-align: middle;
padding: 2px;
}
td.schema-alignment-node-details, td.schema-alignment-link-details {
width: 90%;
}
a.schema-alignment-node-tag {
padding: 3px 6px;
background: #ddd;
text-decoration: none;
color: black;
-moz-border-radius: 10px;
}
a.schema-alignment-node-tag:hover {
background: #888;
color: white;
}
.schema-alignment-node-column {
font-weight: bold;
}
a.schema-alignment-link-tag {
padding: 3px 6px;
text-decoration: none;
color: black;
}
a.schema-alignment-link-tag:hover {
color: #88f;
}
div.schema-alignment-dialog-preview {
height: 400px;
overflow: auto;
background: white;
padding: 10px;
margin-top: 3px;
white-space: pre;
font-family: monospace;
font-size: 9pt;
}
.schema-alignment-status-indicator {
color: #red;
}
/*--------------------------------------------------
* Node dialog
*--------------------------------------------------
*/
.schema-align-node-dialog-node-type {
padding: 0.25em;
background: #ddf;
}
.schema-align-node-dialog-node-type input {
vertical-align: text-bottom;
}
.schema-alignment-node-dialog-column-list {
height: 300px;
width: 200px;
overflow: auto;
border: 1px solid #ddd;
}
/*--------------------------------------------------
* Link dialog
*--------------------------------------------------
*/
.schema-alignment-link-menu-section {
padding: 8px;
border-bottom: 1px solid #ddd;
}
.schema-alignment-link-menu-section-last {
padding: 8px;
}

34
extensions/freebase/module/styles/theme.less
View File

@ -1,34 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../../../../main/webapp/modules/core/styles/theme.less");

42
extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java
View File

@ -1,42 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase;
public class FreebaseProperty extends FreebaseTopic {
//final protected FreebaseType _expectedType;
public FreebaseProperty(String id, String name) {
super(id, name);
}
}

62
extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java
View File

@ -1,62 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
public class FreebaseTopic implements Jsonizable {
final public String id;
final public String name;
public FreebaseTopic(String id, String name) {
this.id = id;
this.name = name;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.endObject();
}
}

70
extensions/freebase/src/com/google/refine/freebase/FreebaseType.java
View File

@ -1,70 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
public class FreebaseType extends FreebaseTopic implements Jsonizable {
public FreebaseType(String id, String name) {
super(id, name);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.endObject();
}
static public FreebaseType load(JSONObject obj) throws Exception {
if (obj == null) {
return null;
}
FreebaseType type = new FreebaseType(
obj.getString("id"),
obj.getString("name")
);
return type;
}
}

110
extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java
View File

@ -1,110 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Properties;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.TransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.Transposer;
import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
import com.google.refine.model.Project;
abstract public class ProtographTransposeExporter implements WriterExporter {
final protected String _contentType;
public ProtographTransposeExporter(String contentType) {
_contentType = contentType;
}
@Override
public String getContentType() {
return "application/x-unknown";
}
public boolean takeWriter() {
return true;
}
public void export(Project project, Properties options, Engine engine,
OutputStream outputStream) throws IOException {
throw new RuntimeException("Not implemented");
}
@Override
public void export(Project project, Properties options, Engine engine,
Writer writer) throws IOException {
Protograph protograph = (Protograph) project.overlayModels.get("freebaseProtograph");
if (protograph != null) {
TransposedNodeFactory nodeFactory = createNodeFactory(project, writer);
Transposer.transpose(project, engine.getAllFilteredRows(),
protograph, protograph.getRootNode(0), nodeFactory, -1);
nodeFactory.flush();
}
}
abstract protected TransposedNodeFactory createNodeFactory(Project project, Writer writer);
static public class TripleLoaderExporter extends ProtographTransposeExporter {
public TripleLoaderExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new TripleLoaderTransposedNodeFactory(project, writer);
}
}
static public class MqlwriteLikeExporter extends ProtographTransposeExporter {
public MqlwriteLikeExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new MqlwriteLikeTransposedNodeFactory(writer);
}
}
}

65
extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java
View File

@ -1,65 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.freebase.operations.ExtendDataOperation;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class ExtendDataCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project,
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String baseColumnName = request.getParameter("baseColumnName");
int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
String jsonString = request.getParameter("extension");
JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
return new ExtendDataOperation(
engineConfig,
baseColumnName,
extension,
columnInsertIndex
);
}
}

69
extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java
View File

@ -1,69 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.freebase.operations.ImportQADataOperation;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.process.Process;
public class ImportQADataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
AbstractOperation op = new ImportQADataOperation();
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

70
extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java
View File

@ -1,70 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
/**
* Perform an MQLread on the server, using the client's credentials.
*
* Currently unused. All client code calls the Freebase API directly.
*/
public class MQLReadCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String query = request.getParameter("query");
@SuppressWarnings("deprecation")
String result = FreebaseUtils.mqlread(provider,query);
response.getWriter().write(result);
} catch (Exception e) {
respondException(response, e);
}
}
}

73
extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java
View File

@ -1,73 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class MQLWriteCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
if (access_credentials != null) {
String query = request.getParameter("query");
String result = FreebaseUtils.mqlwrite(access_credentials, provider, query);
response.getWriter().write(result);
} else {
respond(response, "401 Unauthorized", "You don't have the right credentials");
}
} catch (Exception e) {
respondException(response, e);
}
}
}

191
extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java
View File

@ -1,191 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseDataExtensionJob;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
public class PreviewExtendDataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String columnName = request.getParameter("columnName");
String rowIndicesString = request.getParameter("rowIndices");
if (rowIndicesString == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }");
return;
}
String jsonString = request.getParameter("extension");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
int length = rowIndices.length();
int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
List<String> topicNames = new ArrayList<String>();
List<String> topicIds = new ArrayList<String>();
Set<String> ids = new HashSet<String>();
for (int i = 0; i < length; i++) {
int rowIndex = rowIndices.getInt(i);
if (rowIndex >= 0 && rowIndex < project.rows.size()) {
Row row = project.rows.get(rowIndex);
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
topicNames.add(cell.recon.match.name);
topicIds.add(cell.recon.match.id);
ids.add(cell.recon.match.id);
} else {
topicNames.add(null);
topicIds.add(null);
ids.add(null);
}
}
}
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("code"); writer.value("ok");
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.key("names");
writer.array();
for (String name : info.names) {
writer.value(name);
}
writer.endArray();
writer.key("path");
writer.array();
for (String id : info.path) {
writer.value(id);
}
writer.endArray();
writer.endObject();
}
writer.endArray();
writer.key("rows");
writer.array();
for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r);
String topicName = topicNames.get(r);
if (id != null && map.containsKey(id)) {
DataExtension ext = map.get(id);
boolean first = true;
if (ext.data.length > 0) {
for (Object[] row : ext.data) {
writer.array();
if (first) {
writer.value(topicName);
first = false;
} else {
writer.value(null);
}
for (Object cell : row) {
if (cell != null && cell instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) cell;
writer.object();
writer.key("id"); writer.value(rc.id);
writer.key("name"); writer.value(rc.name);
writer.endObject();
} else {
writer.value(cell);
}
}
writer.endArray();
}
continue;
}
}
writer.array();
if (id != null) {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(topicName);
writer.endObject();
} else {
writer.value("<not reconciled>");
}
writer.endArray();
}
writer.endArray();
writer.endObject();
} catch (Exception e) {
respondException(response, e);
}
}
}

104
extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java
View File

@ -1,104 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.commands.Command;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.Transposer;
import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class PreviewProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
FilteredRows filteredRows = engine.getAllFilteredRows();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
StringBuffer sb = new StringBuffer(2048);
sb.append("{ ");
{
StringWriter stringWriter = new StringWriter();
TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append("\"tripleloader\" : ");
sb.append(JSONObject.quote(stringWriter.toString()));
}
{
StringWriter stringWriter = new StringWriter();
MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append(", \"mqllike\" : ");
sb.append(stringWriter.toString());
}
sb.append(" }");
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

73
extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java
View File

@ -1,73 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.commands.Command;
import com.google.refine.freebase.operations.SaveProtographOperation;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.process.Process;
import com.google.refine.util.ParsingUtilities;
public class SaveProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
AbstractOperation op = new SaveProtographOperation(protograph);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
}
}
}

122
extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java
View File

@ -1,122 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
public class UploadDataCommand extends Command {
final static public String s_dataLoadJobIDPref = "freebase.load.jobID";
final static public String s_dataLoadJobNamePref = "freebase.load.jobName";
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore();
TripleLoaderExporter exporter = new TripleLoaderExporter();
StringWriter triples = new StringWriter(10 * 1024 * 1024);
exporter.export(project, new Properties(), engine, triples);
String source_name = request.getParameter("source_name");
String source_id = request.getParameter("source_id");
String qa = request.getParameter("qa");
String mdo_id = null;
preferenceStore.put(s_dataLoadJobNamePref, source_name);
try {
Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref);
if (jobID != null) {
URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID);
String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
if (!s.equals("null")) {
mdo_id = s;
}
}
} catch (Exception e) {
// ignore
}
String uploadResponse = FreebaseUtils.uploadTriples(
request, qa, source_name, source_id, mdo_id, triples.toString()
);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
try {
JSONObject obj = new JSONObject(uploadResponse);
if (obj.has("result") && !obj.isNull("result")) {
JSONObject result = obj.getJSONObject("result");
if (result.has("job_id") && !result.isNull("job_id")) {
Integer jobID = result.getInt("job_id");
project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
}
}
response.getWriter().write(uploadResponse);
} catch (JSONException e) {
respond(response,"500 Error", uploadResponse);
}
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

81
extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java
View File

@ -1,81 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class CheckAuthorizationCommand extends Command {
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
// this cookie should not be there, but this is good hygiene practice
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
if (access_credentials != null) {
String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
response.getWriter().write(user_info);
} else {
respond(response, "401 Unauthorized", "You don't have the right credentials");
}
} catch (Exception e) {
logger.info("error",e);
respondException(response, e);
}
}
}

70
extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java
View File

@ -1,70 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class GetUserBadgesCommand extends Command {
final static Logger logger = LoggerFactory.getLogger("get-version_command");
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
String user_id = request.getParameter("user_id");
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
response.getWriter().write(user_badges);
} catch (Exception e) {
logger.info("error",e);
respondException(response, e);
}
}
}

125
extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyQuote.java
View File

@ -1,125 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.expr;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.grel.Function;
public class MqlKeyQuote implements Function {
final static private String keyStartChar = "A-Za-z0-9";
final static private String keyInternalChar = "A-Za-z0-9_-";
final static private String keyEndChar = "A-Za-z0-9_";
final static private String fullValidKey = "^[" + keyStartChar + "][" + keyInternalChar + "]*[" + keyEndChar + "]$";
final static private String keyCharMustQuote = "([^" + keyInternalChar + "])";
final static private Pattern fullValidKeyPattern = Pattern.compile(fullValidKey);
final static private Pattern keyCharMustQuotePattern = Pattern.compile(keyCharMustQuote);
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 1) {
Object o1 = args[0];
if (o1 != null && o1 instanceof String) {
return mqlKeyQuote((String) o1);
}
}
return null;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value("Unquotes a MQL key");
writer.key("params"); writer.value("string s");
writer.key("returns"); writer.value("string");
writer.endObject();
}
static String mqlKeyQuote(String s) {
if (fullValidKeyPattern.matcher(s).find()) {
return s;
}
StringBuffer sb = new StringBuffer();
int last = 0;
Matcher m = keyCharMustQuotePattern.matcher(s);
while (m.find()) {
int start = m.start();
int end = m.end();
if (start > last) {
sb.append(s.substring(last, start));
}
last = end;
sb.append('$');
sb.append(quote(s.charAt(start)));
}
if (last < s.length()) {
sb.append(s.substring(last));
}
if (sb.length() > 0) {
if (sb.charAt(0) == '-' || sb.charAt(0) == '_') {
char c = sb.charAt(0);
sb.deleteCharAt(0);
sb.insert(0, '$');
sb.insert(1, quote(c));
}
int length = sb.length();
if (sb.charAt(length-1) == '-') {
sb.deleteCharAt(length-1);
sb.insert(length-1, '$');
sb.insert(length, quote('-'));
}
}
return sb.toString();
}
static String quote(char c) {
return StringUtils.leftPad(Integer.toHexString(c).toUpperCase(), 4, '0');
}
}

89
extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyUnquote.java
View File

@ -1,89 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.expr;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.grel.Function;
public class MqlKeyUnquote implements Function {
final static private Pattern quotedCharPattern = Pattern.compile("\\$([0-9A-Fa-f]{4})");
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 1) {
Object o1 = args[0];
if (o1 != null && o1 instanceof String) {
String s = (String) o1;
StringBuffer sb = new StringBuffer();
int last = 0;
Matcher m = quotedCharPattern.matcher(s);
while (m.find()) {
int start = m.start();
int end = m.end();
if (start > last) {
sb.append(s.substring(last, start));
}
last = end;
sb.append((char)Integer.parseInt(s.substring(start + 1, end), 16));
}
if (last < s.length()) {
sb.append(s.substring(last));
}
return sb.toString();
}
}
return null;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value("Quotes a string into a MQL key");
writer.key("params"); writer.value("string s");
writer.key("returns"); writer.value("string");
writer.endObject();
}
}

469
extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java
View File

@ -1,469 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Serializable;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.freebase.model.recon.DataExtensionReconConfig;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.history.Change;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class DataExtensionChange implements Change {
final protected String _baseColumnName;
final protected int _columnInsertIndex;
final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes;
final protected List<Integer> _rowIndices;
final protected List<DataExtension> _dataExtensions;
protected long _historyEntryID;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
public DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
long historyEntryID
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_historyEntryID = historyEntryID;
}
protected DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_firstNewCellIndex = firstNewCellIndex;
_oldRows = oldRows;
_newRows = newRows;
}
@Override
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_oldRows = new ArrayList<Row>(project.rows);
_newRows = new ArrayList<Row>(project.rows.size());
int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
int index = 0;
int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) {
if (r2 < project.rows.size()) {
Row oldRow2 = project.rows.get(r2);
if (oldRow2.isCellBlank(cellIndex) &&
oldRow2.isCellBlank(keyCellIndex)) {
Row newRow = oldRow2.dup();
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
r2++;
continue;
}
}
Row newRow = new Row(cellIndex + _columnNames.size());
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
}
}
project.rows.clear();
project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
try {
project.columnModel.addColumn(_columnInsertIndex + i, column, true);
// the column might have been renamed to avoid collision
_columnNames.set(i, column.getName());
} catch (ModelException e) {
// won't get here since we set the avoid collision flag
}
}
project.update();
}
}
protected void extendRow(
Row row,
DataExtension dataExtension,
int extensionRowIndex,
Map<String, Recon> reconMap
) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.id)) {
recon = reconMap.get(rc.id);
} else {
recon = Recon.makeFreebaseRecon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "mql";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.id, recon);
}
cell = new Cell(rc.name, recon);
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
@Override
public void revert(Project project) {
synchronized (project) {
project.rows.clear();
project.rows.addAll(_oldRows);
for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnInsertIndex);
}
project.update();
}
}
@Override
public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) {
writer.write(name); writer.write('\n');
}
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (FreebaseType type : _columnTypes) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
type.write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
writer.write('\n');
}
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
if (dataExtension == null) {
writer.write('\n');
continue;
}
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
for (Row row : _oldRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null;
int columnInsertIndex = -1;
List<String> columnNames = null;
List<FreebaseType> columnTypes = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Row> oldRows = null;
List<Row> newRows = null;
int firstNewCellIndex = -1;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) {
baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value);
rowIndices = new ArrayList<Integer>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
rowIndices.add(Integer.parseInt(line));
}
}
} else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value);
columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
columnNames.add(line);
}
}
} else if ("columnTypeCount".equals(field)) {
int count = Integer.parseInt(value);
columnTypes = new ArrayList<FreebaseType>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
}
} else if ("dataExtensionCount".equals(field)) {
int count = Integer.parseInt(value);
dataExtensions = new ArrayList<DataExtension>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line == null) {
continue;
}
if (line.length() == 0) {
dataExtensions.add(null);
continue;
}
int rowCount = Integer.parseInt(line);
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
}
} else if ("oldRowCount".equals(field)) {
int count = Integer.parseInt(value);
oldRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line, pool));
}
}
} else if ("newRowCount".equals(field)) {
int count = Integer.parseInt(value);
newRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line, pool));
}
}
}
}
DataExtensionChange change = new DataExtensionChange(
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows
);
return change;
}
}

99
extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java
View File

@ -1,99 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.recon;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
public class DataExtensionReconConfig extends StrictReconConfig {
final public FreebaseType type;
private final static String WARN = "Not implemented";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject type = obj.getJSONObject("type");
return new DataExtensionReconConfig(
new FreebaseType(
type.getString("id"),
type.getString("name")
)
);
}
public DataExtensionReconConfig(FreebaseType type) {
this.type = type;
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
throw new RuntimeException(WARN);
}
@Override
public int getBatchSize() {
throw new RuntimeException(WARN);
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("mode"); writer.value("extend");
writer.key("type"); type.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
throw new RuntimeException(WARN);
}
@Override
public String getBriefDescription(Project project, String columnName) {
throw new RuntimeException(WARN);
}
}

205
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
View File

@ -1,205 +0,0 @@
/*
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class GuidBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new GuidBasedReconConfig();
}
public GuidBasedReconConfig() {
}
static protected class GuidBasedReconJob extends ReconJob {
String guid;
@Override
public String getStringKey() {
return guid;
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
GuidBasedReconJob job = new GuidBasedReconJob();
String s = cell.value.toString();
if (s.startsWith("/guid/")) {
s = "#" + s.substring(6);
} else if (!s.startsWith("#")) {
s = "#" + s;
}
job.guid = s;
return job;
}
@Override
public int getBatchSize() {
return 50;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
try {
String query = buildQuery(jobs);
String s = FreebaseUtils.mqlread(query);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String guid = result.getString("guid");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
}
}
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {
String guid = ((GuidBasedReconJob) job).guid;
Recon recon = guidToRecon.get(guid);
if (recon == null) {
recon = createNoMatchRecon(historyEntryID);
}
recons.add(recon);
}
return recons;
}
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("guid|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((GuidBasedReconJob) job).guid);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
return query;
}
}

210
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
View File

@ -1,210 +0,0 @@
/*
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class IdBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new IdBasedReconConfig();
}
public IdBasedReconConfig() {
}
static protected class IdBasedReconJob extends ReconJob {
String id;
@Override
public String getStringKey() {
return id;
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
IdBasedReconJob job = new IdBasedReconJob();
String s = cell.value.toString();
if (!s.startsWith("/")) {
if (s.startsWith("92")) {
s = "/guid/" + s;
} else if (!s.contains("/")){
s = "/en/" + s;
} else {
s = "/" + s;
}
}
job.id = s;
return job;
}
@Override
public int getBatchSize() {
return 40;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> idToRecon = new HashMap<String, Recon>();
try {
String query = buildQuery(jobs);
String s = FreebaseUtils.mqlread(query);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String id = result.getString("id");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
}
}
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {
String id = ((IdBasedReconJob) job).id;
Recon recon = idToRecon.get(id);
if (recon == null) {
recon = createNoMatchRecon(historyEntryID);
}
recons.add(recon);
}
return recons;
}
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("id|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((IdBasedReconJob) job).id);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
}
return query;
}
}

224
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
View File

@ -1,224 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class KeyBasedReconConfig extends StrictReconConfig {
final public FreebaseTopic namespace;
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject ns = obj.getJSONObject("namespace");
return new KeyBasedReconConfig(
new FreebaseTopic(
ns.getString("id"),
ns.getString("name")
)
);
}
public KeyBasedReconConfig(FreebaseTopic namespace) {
this.namespace = namespace;
}
static protected class KeyBasedReconJob extends ReconJob {
String key;
@Override
public String getStringKey() {
return key;
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
KeyBasedReconJob job = new KeyBasedReconJob();
job.key = cell.value.toString().replace(' ', '_');
return job;
}
@Override
public int getBatchSize() {
return 40;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("key");
writer.key("namespace"); namespace.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
try {
String query = buildQuery(jobs);
String s = FreebaseUtils.mqlread(query);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
}
}
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {
String key = ((KeyBasedReconJob) job).key;
Recon recon = keyToRecon.get(key);
if (recon == null) {
recon = createNoMatchRecon(historyEntryID);
}
recons.add(recon);
}
return recons;
}
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("key");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("namespace");
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(namespace.id);
jsonWriter.endObject();
jsonWriter.key("value"); jsonWriter.value(null);
jsonWriter.key("value|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((KeyBasedReconJob) job).key);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
}
return query;
}
}

68
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -1,68 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.model.recon;
import org.json.JSONObject;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match");
if ("key".equals(match)) {
return KeyBasedReconConfig.reconstruct(obj);
} else if ("id".equals(match)) {
return IdBasedReconConfig.reconstruct(obj);
} else if ("guid".equals(match)) {
return GuidBasedReconConfig.reconstruct(obj);
}
return null;
}
@Override
public Recon createNewRecon(long historyEntryID) {
return Recon.makeFreebaseRecon(historyEntryID);
}
protected Recon createNoMatchRecon(long historyEntryID) {
Recon recon = createNewRecon(historyEntryID);
recon.service = "mql";
recon.judgment = Judgment.None;
recon.matchRank = -1;
return recon;
}
}

77
extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java
View File

@ -1,77 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.oauth;
import oauth.signpost.OAuthConsumer;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class FreebaseProvider extends Provider {
static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"};
static public void register() {
OAuthUtilities.registerOAuthProvider(new FreebaseProvider(FreebaseUtils.FREEBASE_HOST), FREEBASE_OAUTH_INFO);
}
public FreebaseProvider(String host) {
super(host);
}
@Override
public String getRequestTokenServiceURL() {
return "http://api." + host + "/api/oauth/request_token";
}
@Override
public String getAccessTokenServiceURL() {
return "http://api." + host + "/api/oauth/access_token";
}
@Override
public String getUserAuthorizationURL() {
return "http://www." + host + "/signin/app";
}
@Override
public String getRealm() {
return "http://api" + host + "/";
}
@Override
public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) {
return new FreebaseTimeCommonsHttpOAuthConsumer(consumerKey, consumerSecret);
}
}

103
extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
View File

@ -1,103 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.oauth;
import java.io.IOException;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer {
final static Logger logger = LoggerFactory.getLogger("oauth");
private static final long serialVersionUID = -4139931605235255279L;
private static final int SOCKET_TIMEOUT = 3000;
private static final int CONNECTION_TIMEOUT = 3000;
private static final String TIMER_URL = "http://refinery.freebaseapps.com/time";
public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) {
super(consumerKey, consumerSecret);
}
/**
* It might be that the user's computer's clock is not synchronized enough with the Freebase servers
* and this might result in Freebase thinking that it was under a replay attack.
* To avoid this problem we get the timestamp directly from acre that we know is synchronized.
*
* NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same
* could be said if we used an NTP client.
*/
@Override
protected String generateTimestamp() {
long time = -1;
try {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT);
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT);
HttpClient httpClient = new DefaultHttpClient(httpParams);
HttpGet httpget = new HttpGet(TIMER_URL);
HttpResponse response = httpClient.execute(httpget);
HttpEntity entity = response.getEntity();
if (entity != null) {
time = Long.parseLong(EntityUtils.toString(entity),10);
logger.debug("Got remote timestamp {}", time);
}
} catch (IOException e) {
logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e);
}
if (time == -1) {
time = System.currentTimeMillis();
}
return Long.toString(time / 1000L);
}
}

315
extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java
View File

@ -1,315 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.operations;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.freebase.model.changes.DataExtensionChange;
import com.google.refine.freebase.util.FreebaseDataExtensionJob;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellAtRow;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
public class ExtendDataOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected JSONObject _extension;
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ExtendDataOperation(
engineConfig,
obj.getString("baseColumnName"),
obj.getJSONObject("extension"),
obj.getInt("columnInsertIndex")
);
}
public ExtendDataOperation(
JSONObject engineConfig,
String baseColumnName,
JSONObject extension,
int columnInsertIndex
) {
super(engineConfig);
_baseColumnName = baseColumnName;
_extension = extension;
_columnInsertIndex = columnInsertIndex;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
writer.key("baseColumnName"); writer.value(_baseColumnName);
writer.key("extension"); writer.value(_extension);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + _baseColumnName;
}
protected String createDescription(Column column, List<CellAtRow> cellsAtRows) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + column.getName() +
" by filling " + cellsAtRows.size();
}
@Override
public Process createProcess(Project project, Properties options) throws Exception {
return new ExtendDataProcess(
project,
getEngineConfig(),
getBriefDescription(null)
);
}
public class ExtendDataProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected JSONObject _engineConfig;
final protected long _historyEntryID;
protected int _cellIndex;
protected FreebaseDataExtensionJob _job;
public ExtendDataProcess(
Project project,
JSONObject engineConfig,
String description
) throws JSONException {
super(description);
_project = project;
_engineConfig = engineConfig;
_historyEntryID = HistoryEntry.allocateID();
_job = new FreebaseDataExtensionJob(_extension);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(hashCode());
writer.key("description"); writer.value(_description);
writer.key("immediate"); writer.value(false);
writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done"));
writer.key("progress"); writer.value(_progress);
writer.endObject();
}
@Override
protected Runnable getRunnable() {
return this;
}
protected void populateRowsWithMatches(List<Integer> rowIndices) throws Exception {
Engine engine = new Engine(_project);
engine.initializeFromJSON(_engineConfig);
Column column = _project.columnModel.getColumnByName(_baseColumnName);
if (column == null) {
throw new Exception("No column named " + _baseColumnName);
}
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
return false;
}
}.init(rowIndices));
}
protected int extendRows(
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
Map<String, ReconCandidate> reconCandidateMap
) {
Set<String> ids = new HashSet<String>();
int end;
for (end = from; end < limit && ids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
ids.add(cell.recon.match.id);
}
Map<String, DataExtension> map = null;
try {
map = _job.extend(ids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.id;
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
return end;
}
@Override
public void run() {
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try {
populateRowsWithMatches(rowIndices);
} catch (Exception e2) {
// TODO : Not sure what to do here?
e2.printStackTrace();
}
int start = 0;
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
_progress = end * 100 / rowIndices.size();
try {
Thread.sleep(200);
} catch (InterruptedException e) {
if (_canceled) {
break;
}
}
}
if (!_canceled) {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) {
columnTypes.add(info.expectedType);
}
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
ExtendDataOperation.this,
new DataExtensionChange(
_baseColumnName,
_columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
_historyEntryID)
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
}
}
}

140
extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java
View File

@ -1,140 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.operations;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.commands.UploadDataCommand;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassReconChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
public class ImportQADataOperation extends AbstractOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new ImportQADataOperation();
}
public ImportQADataOperation() {
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Import QA DAta";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref);
if (jobID == null) {
throw new InternalError("Project is not associated with any data loading job.");
}
Map<Long, String> reconIDToResult = new HashMap<Long, String>();
URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(30000); // 30 seconds
LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream()));
try {
String line;
while ((line = reader.readLine()) != null) {
JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line);
long reconID = Long.parseLong(obj.getString("recon_id").substring(3));
reconIDToResult.put(reconID, obj.getString("result"));
}
} finally {
reader.close();
}
Map<Long, Recon> oldRecons = new HashMap<Long, Recon>();
Map<Long, Recon> newRecons = new HashMap<Long, Recon>();
for (int r = 0; r < project.rows.size(); r++) {
Row row = project.rows.get(r);
for (int c = 0; c < row.cells.size(); c++) {
Cell cell = row.cells.get(c);
if (cell != null && cell.recon != null) {
Recon oldRecon = cell.recon;
if (reconIDToResult.containsKey(oldRecon.id)) {
Recon newRecon = oldRecon.dup();
newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id));
reconIDToResult.remove(oldRecon.id);
oldRecons.put(oldRecon.id, oldRecon);
newRecons.put(oldRecon.id, newRecon);
}
}
}
}
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(project),
this,
new MassReconChange(newRecons, oldRecons)
);
}
}

159
extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java
View File

@ -1,159 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveProtographOperation extends AbstractOperation {
final protected Protograph _protograph;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new SaveProtographOperation(
Protograph.reconstruct(obj.getJSONObject("protograph"))
);
}
public SaveProtographOperation(Protograph protograph) {
_protograph = protograph;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("protograph"); _protograph.write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Save schema alignment skeleton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Change change = new ProtographChange(_protograph);
return new HistoryEntry(historyEntryID, project, getBriefDescription(project), SaveProtographOperation.this, change);
}
static public class ProtographChange implements Change {
final protected Protograph _newProtograph;
protected Protograph _oldProtograph;
public ProtographChange(Protograph protograph) {
_newProtograph = protograph;
}
@Override
public void apply(Project project) {
synchronized (project) {
_oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph");
project.overlayModels.put("freebaseProtograph", _newProtograph);
}
}
@Override
public void revert(Project project) {
synchronized (project) {
if (_oldProtograph == null) {
project.overlayModels.remove("freebaseProtograph");
} else {
project.overlayModels.put("freebaseProtograph", _oldProtograph);
}
}
}
@Override
public void save(Writer writer, Properties options) throws IOException {
writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
Protograph oldProtograph = null;
Protograph newProtograph = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldProtograph".equals(field) && value.length() > 0) {
oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newProtograph".equals(field) && value.length() > 0) {
newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
ProtographChange change = new ProtographChange(newProtograph);
change._oldProtograph = oldProtograph;
return change;
}
static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
if (p != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
p.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

84
extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java
View File

@ -1,84 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
public class AnonymousNode implements Node, NodeWithLinks {
final public FreebaseType type;
final public List<Link> links = new LinkedList<Link>();
public AnonymousNode(FreebaseType type) {
this.type = type;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("anonymous");
writer.key("type"); type.write(writer, options);
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@Override
public void addLink(Link link) {
links.add(link);
}
@Override
public Link getLink(int index) {
return links.get(index);
}
@Override
public int getLinkCount() {
return links.size();
}
}

75
extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java
View File

@ -1,75 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
public class BooleanColumnCondition implements Condition {
final public String columnName;
public BooleanColumnCondition(String columnName) {
this.columnName = columnName;
}
@Override
public boolean test(Project project, int rowIndex, Row row) {
Column column = project.columnModel.getColumnByName(columnName);
if (column != null) {
Object o = row.getCellValue(column.getCellIndex());
if (o != null) {
if (o instanceof Boolean) {
return ((Boolean) o).booleanValue();
} else {
return Boolean.parseBoolean(o.toString());
}
}
}
return false;
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("columnName"); writer.value(columnName);
writer.endObject();
}
}

69
extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java
View File

@ -1,69 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
public class CellKeyNode extends CellNode {
final public FreebaseTopic namespace;
public CellKeyNode(
FreebaseTopic namespace
) {
this.namespace = namespace;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-key");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
writer.key("namespace"); namespace.write(writer, options);
writer.endObject();
}
}

41
extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java
View File

@ -1,41 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
abstract public class CellNode implements Node {
final public List<String> columnNames = new LinkedList<String>();
}

95
extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java
View File

@ -1,95 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
public class CellTopicNode extends CellNode implements NodeWithLinks {
final public FreebaseType type;
final public List<Link> links = new LinkedList<Link>();
public CellTopicNode(
FreebaseType type
) {
this.type = type;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-topic");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
if (type != null) {
writer.key("type"); type.write(writer, options);
}
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@Override
public void addLink(Link link) {
links.add(link);
}
@Override
public Link getLink(int index) {
return links.get(index);
}
@Override
public int getLinkCount() {
return links.size();
}
}

70
extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java
View File

@ -1,70 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
public class CellValueNode extends CellNode {
final public String valueType;
final public String lang;
public CellValueNode(
String valueType,
String lang
) {
this.valueType = valueType;
this.lang = lang;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-value");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
writer.key("valueType"); writer.value(valueType);
writer.key("lang"); writer.value(lang);
writer.endObject();
}
}

42
extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java
View File

@ -1,42 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import com.google.refine.Jsonizable;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
public interface Condition extends Jsonizable {
public boolean test(Project project, int rowIndex, Row row);
}

85
extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java
View File

@ -1,85 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
public class FreebaseTopicNode implements Node, NodeWithLinks {
final public FreebaseTopic topic;
final public List<Link> links = new LinkedList<Link>();
public FreebaseTopicNode(FreebaseTopic topic) {
this.topic = topic;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("topic");
writer.key("topic"); topic.write(writer, options);
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@Override
public void addLink(Link link) {
links.add(link);
}
@Override
public Link getLink(int index) {
return links.get(index);
}
@Override
public int getLinkCount() {
return links.size();
}
}

82
extensions/freebase/src/com/google/refine/freebase/protograph/Link.java
View File

@ -1,82 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.freebase.FreebaseProperty;
public class Link implements Jsonizable {
final public FreebaseProperty property;
final public Node target;
final public Condition condition;
final public boolean load;
public Link(FreebaseProperty property, Node target, Condition condition, boolean load) {
this.property = property;
this.target = target;
this.condition = condition;
this.load = load;
}
public FreebaseProperty getProperty() {
return property;
}
public Node getTarget() {
return target;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("property"); property.write(writer, options);
if (target != null) {
writer.key("target");
target.write(writer, options);
}
if (condition != null) {
writer.key("condition");
condition.write(writer, options);
}
writer.endObject();
}
}

39
extensions/freebase/src/com/google/refine/freebase/protograph/Node.java
View File

@ -1,39 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import com.google.refine.Jsonizable;
public interface Node extends Jsonizable {
}

42
extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java
View File

@ -1,42 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
public interface NodeWithLinks {
public void addLink(Link link);
public int getLinkCount();
public Link getLink(int index);
}

205
extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java
View File

@ -1,205 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
public class Protograph implements OverlayModel {
final protected List<Node> _rootNodes = new LinkedList<Node>();
public int getRootNodeCount() {
return _rootNodes.size();
}
public Node getRootNode(int index) {
return _rootNodes.get(index);
}
@Override
public void onBeforeSave(Project project) {
}
@Override
public void onAfterSave(Project project) {
}
@Override
public void dispose(Project project) {
}
static public Protograph reconstruct(JSONObject o) throws JSONException {
Protograph g = new Protograph();
JSONArray rootNodes = o.getJSONArray("rootNodes");
int count = rootNodes.length();
for (int i = 0; i < count; i++) {
JSONObject o2 = rootNodes.getJSONObject(i);
Node node = reconstructNode(o2);
if (node != null) {
g._rootNodes.add(node);
}
}
return g;
}
static protected Node reconstructNode(JSONObject o) throws JSONException {
Node node = null;
String nodeType = o.getString("nodeType");
if (nodeType.startsWith("cell-as-")) {
if ("cell-as-topic".equals(nodeType)) {
if (o.has("type")) {
node = new CellTopicNode(
reconstructType(o.getJSONObject("type"))
);
}
} else if ("cell-as-value".equals(nodeType)) {
node = new CellValueNode(
o.getString("valueType"),
o.getString("lang")
);
} else if ("cell-as-key".equals(nodeType)) {
node = new CellKeyNode(
reconstructTopic(o.getJSONObject("namespace"))
);
}
if (o.has("columnName") && !o.isNull("columnName")) {
((CellNode) node).columnNames.add(o.getString("columnName"));
}
if (o.has("columnNames") && !o.isNull("columnNames")) {
JSONArray columnNames = o.getJSONArray("columnNames");
int count = columnNames.length();
for (int c = 0; c < count; c++) {
((CellNode) node).columnNames.add(columnNames.getString(c));
}
}
} else if ("topic".equals(nodeType)) {
node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
} else if ("value".equals(nodeType)) {
node = new ValueNode(
o.get("value"),
o.getString("valueType"),
o.getString("lang")
);
} else if ("anonymous".equals(nodeType)) {
node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
}
if (node != null && node instanceof NodeWithLinks && o.has("links")) {
NodeWithLinks node2 = (NodeWithLinks) node;
JSONArray links = o.getJSONArray("links");
int linkCount = links.length();
for (int j = 0; j < linkCount; j++) {
JSONObject oLink = links.getJSONObject(j);
Condition condition = null;
if (oLink.has("condition") && !oLink.isNull("condition")) {
JSONObject oCondition = oLink.getJSONObject("condition");
if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
condition = new BooleanColumnCondition(oCondition.getString("columnName"));
}
}
node2.addLink(new Link(
reconstructProperty(oLink.getJSONObject("property")),
oLink.has("target") && !oLink.isNull("target") ?
reconstructNode(oLink.getJSONObject("target")) : null,
condition,
oLink.has("load") && !oLink.isNull("load") ?
oLink.getBoolean("load") : true
));
}
}
return node;
}
static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
return new FreebaseProperty(
o.getString("id"),
o.getString("name")
);
}
static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
return new FreebaseType(
o.getString("id"),
o.getString("name")
);
}
static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
return new FreebaseTopic(
o.getString("id"),
o.getString("name")
);
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("rootNodes"); writer.array();
for (Node node : _rootNodes) {
node.write(writer, options);
}
writer.endArray();
writer.endObject();
}
static public Protograph load(Project project, JSONObject obj) throws Exception {
return reconstruct(obj);
}
}

63
extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java
View File

@ -1,63 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
public class ValueNode implements Node {
final public Object value;
final public String valueType;
final public String lang;
public ValueNode(Object value, String valueType, String lang) {
this.value = value;
this.valueType = valueType;
this.lang = lang;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("value");
writer.key("value"); writer.value(value);
writer.key("valueType"); writer.value(valueType);
writer.key("lang"); writer.value(lang);
writer.endObject();
}
}

397
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java
View File

@ -1,397 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellKeyNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.CellValueNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Recon;
import com.google.refine.util.JSONUtilities;
public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
protected Writer writer;
protected List<JSONObject> rootObjects = new LinkedList<JSONObject>();
private static final String TYPE = "type";
private static final String ID = "id";
private static final String NAME = "name";
private static final String CREATE = "create";
private static final String VALUE = "value";
private static final String CONNECT = "connect";
private static final String LANG = "lang";
public MqlwriteLikeTransposedNodeFactory(Writer writer) {
this.writer = writer;
}
protected JSONArray getJSON() {
return new JSONArray(rootObjects);
}
@Override
public void flush() throws IOException {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.array();
for (JSONObject obj : rootObjects) {
jsonWriter.value(obj);
}
jsonWriter.endArray();
} catch (JSONException e) {
e.printStackTrace();
}
writer.flush();
}
abstract protected class JsonTransposedNode implements TransposedNode {
abstract public Object getJSON();
}
abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
abstract public JSONObject getJSONObject();
protected JSONObject obj;
@Override
public Object getJSON() {
return getJSONObject();
}
}
protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
JsonObjectTransposedNode parent;
FreebaseProperty property;
AnonymousNode node;
protected AnonymousTransposedNode(
JsonObjectTransposedNode parent,
FreebaseProperty property,
AnonymousNode node
) {
this.parent = parent;
this.property = property;
this.node = node;
}
@Override
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(TYPE, this.node.type.id);
obj.put(ID, (String) null);
obj.put(CREATE, "unconditional");
} catch (JSONException e) {
e.printStackTrace();
}
linkTransposedNodeJSON(obj, parent, property);
}
return obj;
}
}
protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
protected CellTopicNode node;
protected Cell cell;
public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
@Override
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
if (cell.recon != null &&
cell.recon.judgment == Recon.Judgment.Matched &&
cell.recon.match != null) {
obj.put(ID, cell.recon.match.id);
} else {
obj.put(ID, (String) null);
obj.put(NAME, cell.value.toString());
obj.put(TYPE, node.type.id);
obj.put(CREATE, "unless_exists");
}
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class CellValueTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected CellValueNode node;
protected Cell cell;
public CellValueTransposedNode(CellValueNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
@Override
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
JSONUtilities.putField(obj, VALUE, cell.value);
obj.put(TYPE, node.valueType);
if ("/type/text".equals(node.valueType)) {
obj.put(LANG, node.lang);
}
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class CellKeyTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected CellKeyNode node;
protected Cell cell;
public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
@Override
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(VALUE, cell.value.toString());
JSONObject nsObj = new JSONObject();
nsObj.put(ID, node.namespace.id);
obj.put("namespace", nsObj);
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class TopicTransposedNode extends JsonObjectTransposedNode {
protected FreebaseTopicNode node;
public TopicTransposedNode(FreebaseTopicNode node) {
this.node = node;
}
@Override
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(ID, node.topic.id);
// TODO: This won't work at the root of the query, so that needs
// to be special cased, but for now one must use a different shaped graph
// (ie move the Freebase topic to someplace other than the root
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class ValueTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected ValueNode node;
public ValueTransposedNode(ValueNode node) {
this.node = node;
}
@Override
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
if ("/type/datetime".equals(node.valueType) && node.value instanceof Long) {
// Special case integers as year-only dates
obj.put(VALUE, node.value.toString());
} else {
obj.put(VALUE, node.value);
}
obj.put(TYPE, node.valueType);
if ("/type/text".equals(node.valueType)) {
obj.put(LANG, node.lang);
}
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
@Override
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex) {
return new AnonymousTransposedNode(
parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
link != null ? link.property : null,
node
);
}
@Override
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell) {
JsonTransposedNode tnode = null;
if (node instanceof CellTopicNode) {
tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
} else if (node instanceof CellValueNode) {
tnode = new CellValueTransposedNode((CellValueNode) node, cell);
} else if (node instanceof CellKeyNode) {
tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
}
if (tnode != null) {
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
}
return tnode;
}
@Override
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node, int rowIndex) {
JsonTransposedNode tnode = new TopicTransposedNode(node);
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
return tnode;
}
@Override
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node, int rowIndex) {
JsonTransposedNode tnode = new ValueTransposedNode(node);
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
return tnode;
}
protected void processTransposedNode(
JsonTransposedNode tnode,
TransposedNode parentNode,
FreebaseProperty property
) {
if (!(tnode instanceof AnonymousTransposedNode)) {
linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
}
}
protected void linkTransposedNodeJSON(
Object obj,
TransposedNode parentNode,
FreebaseProperty property
) {
if (parentNode == null) {
if (obj instanceof JSONObject) {
rootObjects.add((JSONObject) obj);
}
} else if (parentNode instanceof JsonTransposedNode) {
JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
try {
JSONArray a = null;
if (parentObj.has(property.id)) {
a = parentObj.getJSONArray(property.id);
} else {
a = new JSONArray();
parentObj.put(property.id, a);
}
a.put(a.length(), obj);
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}

103
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/SchemaHelper.java
View File

@ -1,103 +0,0 @@
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
public class SchemaHelper {
static private final String helperURL = "http://openrefine-helper.freebaseapps.com/";
static private class PropertyInfo {
String fromTypeID;
String toTypeID;
}
static private class TypeInfo {
String[] includedTypes;
}
protected Map<String, PropertyInfo> properties = new HashMap<String, PropertyInfo>();
protected Map<String, TypeInfo> types = new HashMap<String, TypeInfo>();
public String getPropertyFromType(String propertyID) {
ensureProperty(propertyID);
return properties.get(propertyID).fromTypeID;
}
public String getPropertyToType(String propertyID) {
ensureProperty(propertyID);
return properties.get(propertyID).toTypeID;
}
public String[] getIncludedTypeIDs(String typeID) {
ensureType(typeID);
return types.get(typeID).includedTypes;
}
private void ensureProperty(String propertyID) {
if (properties.containsKey(propertyID)) {
return;
}
PropertyInfo info = new PropertyInfo();
properties.put(propertyID, info);
JSONObject obj = getJson(helperURL + "get_property_data" + propertyID);
if (obj != null) {
try {
if (!obj.isNull("from")) {
info.fromTypeID = obj.getString("from");
}
} catch (JSONException e) {
}
try {
if (!obj.isNull("to")) {
info.toTypeID = obj.getString("to");
}
} catch (JSONException e) {
}
}
}
private void ensureType(String typeID) {
if (types.containsKey(typeID)) {
return;
}
TypeInfo info = new TypeInfo();
types.put(typeID, info);
JSONObject obj = getJson(helperURL + "get_type_data" + typeID);
if (obj != null) {
if (!obj.isNull("includes")) {
info.includedTypes = JSONUtilities.getStringArray(obj, "includes");
}
}
}
private JSONObject getJson(String urlString) {
try {
URL url = new URL(urlString);
InputStream is = url.openStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
return ParsingUtilities.evaluateJsonStringToObject(s);
} finally {
is.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}

37
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java
View File

@ -1,37 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph.transpose;
public interface TransposedNode {
}

76
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java
View File

@ -1,76 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
public interface TransposedNodeFactory {
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex
);
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell
);
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node,
int rowIndex
);
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node,
int rowIndex
);
public void flush() throws IOException;
}

317
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java
View File

@ -1,317 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph.transpose;
import java.util.LinkedList;
import java.util.List;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.Node;
import com.google.refine.freebase.protograph.NodeWithLinks;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.Row;
public class Transposer {
static public void transpose(
Project project,
FilteredRows filteredRows,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory
) {
transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
}
static public void transpose(
Project project,
FilteredRows filteredRows,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory,
int limit
) {
Context rootContext = new Context(rootNode, null, null, limit);
filteredRows.accept(project, new RowVisitor() {
Context rootContext;
Protograph protograph;
Node rootNode;
TransposedNodeFactory nodeFactory;
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
}
if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
return true;
}
return false;
}
@Override
public void start(Project project) {
}
@Override
public void end(Project project) {
}
public RowVisitor init(
Context rootContext,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory
) {
this.rootContext = rootContext;
this.protograph = protograph;
this.rootNode = rootNode;
this.nodeFactory = nodeFactory;
return this;
}
}.init(rootContext, protograph, rootNode, nodeFactory));
}
static protected void descend(
Project project,
Protograph protograph,
TransposedNodeFactory nodeFactory,
int rowIndex,
Row row,
Node node,
Context context
) {
List<TransposedNode> tnodes = new LinkedList<TransposedNode>();
Link link = context.parent == null ? null : context.link;
if (node instanceof CellNode) {
if (!descendCellNode(project, nodeFactory, rowIndex, row, node, context, tnodes, link)) {
return;
}
} else if (node instanceof AnonymousNode) {
descendAnonymousNode(nodeFactory, rowIndex, node, context, tnodes, link);
} else if (node instanceof FreebaseTopicNode) {
descendFreebaseTopicNode(nodeFactory, rowIndex, node, context, tnodes, link);
} else if (node instanceof ValueNode) {
descendValueNode(nodeFactory, rowIndex, node, context, tnodes, link);
}
if (tnodes.size() > 0) {
context.transposedNodes.clear();
context.transposedNodes.addAll(tnodes);
}
if (node instanceof NodeWithLinks) {
NodeWithLinks node2 = (NodeWithLinks) node;
int linkCount = node2.getLinkCount();
for (int i = 0; i < linkCount; i++) {
Link link2 = node2.getLink(i);
if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
descend(
project,
protograph,
nodeFactory,
rowIndex,
row,
link2.getTarget(),
context.subContexts.get(i)
);
}
}
}
}
private static boolean descendCellNode(Project project, TransposedNodeFactory nodeFactory, int rowIndex, Row row,
Node node, Context context, List<TransposedNode> tnodes, Link link) {
CellNode node2 = (CellNode) node;
for (String columnName : node2.columnNames) {
Column column = project.columnModel.getColumnByName(columnName);
if (column != null) {
int cellIndex = column.getCellIndex();
Cell cell = row.getCell(cellIndex);
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
if (node2 instanceof CellTopicNode &&
(cell.recon == null || cell.recon.judgment == Judgment.None)) {
return false;
}
context.count++;
if (context.limit > 0 && context.count > context.limit) {
return false;
}
if (context.parent == null) {
tnodes.add(nodeFactory.transposeCellNode(
null,
link,
node2,
rowIndex,
cellIndex,
cell
));
} else {
for (TransposedNode parentNode : context.parent.transposedNodes) {
tnodes.add(nodeFactory.transposeCellNode(
parentNode,
link,
node2,
rowIndex,
cellIndex,
cell
));
}
}
}
}
}
return true;
}
private static void descendAnonymousNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node,
Context context, List<TransposedNode> tnodes, Link link) {
if (context.parent == null) {
tnodes.add(nodeFactory.transposeAnonymousNode(
null,
link,
(AnonymousNode) node,
rowIndex
));
} else {
for (TransposedNode parentNode : context.parent.transposedNodes) {
tnodes.add(nodeFactory.transposeAnonymousNode(
parentNode,
link,
(AnonymousNode) node,
rowIndex
));
}
}
}
private static void descendFreebaseTopicNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node,
Context context, List<TransposedNode> tnodes, Link link) {
if (context.parent == null) {
tnodes.add(nodeFactory.transposeTopicNode(
null,
link,
(FreebaseTopicNode) node,
rowIndex
));
} else {
for (TransposedNode parentNode : context.parent.transposedNodes) {
tnodes.add(nodeFactory.transposeTopicNode(
parentNode,
link,
(FreebaseTopicNode) node,
rowIndex
));
}
}
}
private static void descendValueNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node, Context context,
List<TransposedNode> tnodes, Link link) {
if (context.parent == null) {
tnodes.add(nodeFactory.transposeValueNode(
null,
link,
(ValueNode) node,
rowIndex
));
} else {
for (TransposedNode parentNode : context.parent.transposedNodes) {
tnodes.add(nodeFactory.transposeValueNode(
parentNode,
link,
(ValueNode) node,
rowIndex
));
}
}
}
static class Context {
List<TransposedNode> transposedNodes = new LinkedList<TransposedNode>();
List<Context> subContexts;
Context parent;
Link link;
int count;
int limit;
Context(Node node, Context parent, Link link, int limit) {
this.parent = parent;
this.link = link;
this.limit = limit;
if (node instanceof NodeWithLinks) {
NodeWithLinks node2 = (NodeWithLinks) node;
int subContextCount = node2.getLinkCount();
subContexts = new LinkedList<Context>();
for (int i = 0; i < subContextCount; i++) {
Link link2 = node2.getLink(i);
subContexts.add(
new Context(link2.getTarget(), this, link2, -1));
}
}
}
public void nullifySubContextNodes() {
if (subContexts != null) {
for (Context context : subContexts) {
context.transposedNodes.clear();
context.nullifySubContextNodes();
}
}
}
}
}

899
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java
View File

@ -1,899 +0,0 @@
/*
Copyright 2010,2012. Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import java.io.Writer;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellKeyNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.CellValueNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.StandardReconConfig;
public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
protected Project project;
protected boolean start = true;
protected Writer writer;
protected WritingTransposedNode lastRootNode;
protected Map<String, Long> varPool = new HashMap<String, Long>();
protected Map<Long, String> newTopicVars = new HashMap<Long, String>();
protected Set<Long> serializedRecons = new HashSet<Long>();
protected long contextID = 0;
protected int contextRowIndex;
protected int contextRefCount = 0;
protected JSONObject contextTreeRoot;
protected SchemaHelper schemaHelper = new SchemaHelper();
protected Map<String, Set<Long>> typeIDToAssertedReconIDs = new HashMap<String, Set<Long>>();
protected Set<Long> getAssertedReconIDSet(String typeID) {
Set<Long> assertedReconIDSet = typeIDToAssertedReconIDs.get(typeID);
if (assertedReconIDSet == null) {
assertedReconIDSet = new HashSet<Long>();
typeIDToAssertedReconIDs.put(typeID, assertedReconIDSet);
}
return assertedReconIDSet;
}
protected void ensureOneTypeAsserted(Recon recon, String typeID) {
Set<Long> assertedReconIDSet = getAssertedReconIDSet(typeID);
if (!assertedReconIDSet.contains(recon.id)) {
assertedReconIDSet.add(recon.id);
String subject = recon.judgment == Judgment.New ? newTopicVars.get(recon.id) : recon.match.id;
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \"type\"");
sb.append(", \"o\" : \""); sb.append(typeID); sb.append('"');
sb.append(" }");
writeLine(sb.toString());
}
}
protected void ensureAllIncludedTypesAsserted(Recon recon, String typeID) {
ensureOneTypeAsserted(recon, typeID);
String[] includedTypeIDs = schemaHelper.getIncludedTypeIDs(typeID);
if (includedTypeIDs != null) {
for (String typeID2 : includedTypeIDs) {
if (!"/type/object".equals(typeID2)) {
ensureOneTypeAsserted(recon, typeID2);
}
}
}
}
protected void ensureFromTypesAsserted(Recon recon, String propertyID) {
String fromTypeID = schemaHelper.getPropertyFromType(propertyID);
if (fromTypeID != null && !"/type/object".equals(fromTypeID)) {
ensureAllIncludedTypesAsserted(recon, fromTypeID);
}
}
protected void ensureToTypesAsserted(Recon recon, String propertyID) {
String toTypeID = schemaHelper.getPropertyToType(propertyID);
if (toTypeID != null && !"/type/object".equals(toTypeID)) {
ensureAllIncludedTypesAsserted(recon, toTypeID);
}
}
public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
this.project = project;
this.writer = writer;
}
@Override
public void flush() throws IOException {
if (lastRootNode != null) {
lastRootNode.write(null, null, project, -1, -1, null);
lastRootNode = null;
writeContextTreeNode();
}
}
protected void writeLine(String line) {
try {
if (start) {
start = false;
} else {
writer.write('\n');
}
writer.write(line);
} catch (IOException e) {
// ignore
}
}
protected void writeRecon(
StringBuffer sb,
Project project,
int rowIndex,
int cellIndex,
Cell cell
) {
Recon recon = cell.recon;
sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
contextRefCount++;
if (!serializedRecons.contains(recon.id)) {
serializedRecons.add(recon.id);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
// qa:sample_group
{
StringBuffer sb2 = new StringBuffer();
sb2.append("{ \"s\" : \"rec");
sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
sb2.append(JSONObject.quote(column.getName()));
sb2.append(", \"ignore\" : true }");
writeLine(sb2.toString());
}
// qa:recon_data
{
StringBuffer sb2 = new StringBuffer();
String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
sb2.append("{ \"s\" : \"rec");
sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
if (recon.judgment == Judgment.Matched) {
sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
}
ReconConfig reconConfig = column.getReconConfig();
if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
}
sb2.append(" } }");
writeLine(sb2.toString());
}
}
}
protected void writeLine(
String subject, String predicate, Object object,
Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
int objectRowIndex, int objectCellIndex, Cell objectCell,
boolean ignore
) {
if (subject != null && object != null) {
String s = object instanceof String ?
JSONObject.quote((String) object) : object.toString();
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : "); sb.append(s);
if (subjectCell != null || objectCell != null) {
sb.append(", \"meta\" : { ");
sb.append("\"recon\" : { ");
if (subjectCell != null) {
sb.append("\"s\" : ");
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
}
if (objectCell != null) {
if (subjectCell != null) {
sb.append(", ");
}
sb.append("\"o\" : ");
writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
}
sb.append(" }");
sb.append(" }");
}
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }");
writeLine(sb.toString());
}
}
protected void writeLine(
String subject, String predicate, Object object, String lang,
Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
boolean ignore
) {
if (subject != null && object != null) {
String s = object instanceof String ?
JSONObject.quote((String) object) : object.toString();
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : "); sb.append(s);
sb.append(", \"lang\" : \""); sb.append(lang); sb.append('"');
if (subjectCell != null) {
sb.append(", \"meta\" : { ");
sb.append("\"recon\" : { ");
sb.append("\"s\" : ");
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
sb.append(" }");
sb.append(" }");
}
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }");
writeLine(sb.toString());
}
}
abstract protected class WritingTransposedNode implements TransposedNode {
JSONObject jsonContextNode;
boolean load;
public Object write(
String subject, String predicate, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
return internalWrite(
subject, predicate, project,
subjectRowIndex, subjectCellIndex, subjectCell);
}
abstract public Object internalWrite(
String subject, String predicate, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
}
abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
public List<Link> links = new LinkedList<Link>();
public List<Integer> rowIndices = new LinkedList<Integer>();
public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
protected void writeChildren(
String subject, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
Recon recon = subjectCell != null && subjectCell.recon != null &&
(subjectCell.recon.judgment == Judgment.Matched || subjectCell.recon.judgment == Judgment.New)
? subjectCell.recon : null;
for (int i = 0; i < children.size(); i++) {
WritingTransposedNode child = children.get(i);
Link link = links.get(i);
String predicate = link.property.id;
if (recon != null) {
ensureFromTypesAsserted(recon, predicate);
}
child.write(subject, predicate, project,
subjectRowIndex, subjectCellIndex, subjectCell);
}
}
}
protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
//protected AnonymousTransposedNode(AnonymousNode node) { }
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
if (children.size() == 0 || subject == null) {
return null;
}
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : { ");
StringBuffer sbRecon = new StringBuffer();
boolean first = true;
boolean firstRecon = true;
if (subjectCell != null && subjectCell.recon != null) {
sbRecon.append("\"s\" : ");
writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
firstRecon = false;
}
for (int i = 0; i < children.size(); i++) {
WritingTransposedNode child = children.get(i);
Link link = links.get(i);
FreebaseProperty property = link.property;
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
if (c != null) {
if (first) {
first = false;
} else {
sb.append(", ");
}
sb.append("\"" + property.id + "\": ");
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
}
if (child instanceof CellTopicTransposedNode) {
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
Recon recon = child2.cell.recon;
if (recon != null &&
(recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
if (firstRecon) {
firstRecon = false;
} else {
sbRecon.append(", ");
}
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
writeRecon(sbRecon, project,
rowIndices.get(i), child2.cellIndex, child2.cell);
}
}
}
sb.append(" }, \"meta\" : { \"recon\" : { ");
sb.append(sbRecon.toString());
sb.append(" } } }");
writeLine(sb.toString());
return null;
}
}
protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
protected CellTopicNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
String id = null;
if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
int objectRowIndex = rowIndex;
int objectCellIndex = cellIndex;
Cell objectCell = cell;
String typeID = node.type.id;
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
ReconConfig reconConfig = column.getReconConfig();
if (reconConfig instanceof StandardReconConfig) {
typeID = ((StandardReconConfig) reconConfig).typeID;
}
if (cell.recon.judgment == Recon.Judgment.Matched) {
id = cell.recon.match.id;
} else if (cell.recon.judgment == Judgment.New) {
if (newTopicVars.containsKey(cell.recon.id)) {
id = newTopicVars.get(cell.recon.id);
} else {
String columnName = column.getName();
long var = 0;
if (varPool.containsKey(columnName)) {
var = varPool.get(columnName);
}
varPool.put(columnName, var + 1);
id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
getAssertedReconIDSet(typeID).add(cell.recon.id);
newTopicVars.put(cell.recon.id, id);
}
} else {
return null;
}
ensureAllIncludedTypesAsserted(cell.recon, typeID);
if (subject != null) {
ensureToTypesAsserted(cell.recon, predicate);
writeLine(subject, predicate, id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
objectRowIndex, objectCellIndex, objectCell, !load);
}
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
}
return id;
}
}
protected class CellValueTransposedNode extends WritingTransposedNode {
protected JSONObject obj;
protected CellValueNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
Object value = cell.value;
if (value != null) {
if ("/type/text".equals(node.valueType)) {
value = value.toString();
if (subject != null) {
writeLine(subject, predicate, value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell, !load);
}
} else {
value = validateValue(value, node.valueType);
if (subject != null && value != null) {
writeLine(subject, predicate, value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
}
}
}
return value;
}
}
protected class CellKeyTransposedNode extends WritingTransposedNode {
protected CellKeyNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
return null;
}
}
protected class TopicTransposedNode extends TransposedNodeWithChildren {
protected FreebaseTopicNode node;
public TopicTransposedNode(FreebaseTopicNode node) {
this.node = node;
}
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, predicate, node.topic.id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
writeChildren(node.topic.id, project, -1, -1, null);
return node.topic.id;
}
}
protected class ValueTransposedNode extends WritingTransposedNode {
protected ValueNode node;
public ValueTransposedNode(ValueNode node) {
this.node = node;
}
@Override
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
Object value = node.value;
if (value != null) {
if ("/type/text".equals(node.valueType)) {
value = value.toString();
if (subject != null) {
writeLine(subject, predicate, value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell, !load);
}
} else {
value = validateValue(value, node.valueType);
if (subject != null && value != null) {
writeLine(subject, predicate, value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
}
}
}
return value;
}
}
@Override
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new AnonymousTransposedNode();
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
null
);
return tnode;
}
@Override
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = null;
if (node instanceof CellTopicNode) {
if (cell.recon != null &&
(cell.recon.judgment == Judgment.Matched ||
cell.recon.judgment == Judgment.New)) {
tnode = new CellTopicTransposedNode(
(CellTopicNode) node, rowIndex, cellIndex, cell);
}
} else if (node instanceof CellValueNode) {
tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
} else if (node instanceof CellKeyNode) {
tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
}
if (tnode != null) {
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
cell,
rowIndex
);
}
return tnode;
}
@Override
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node,
int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new TopicTransposedNode(node);
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
node.topic
);
return tnode;
}
@Override
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node,
int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new ValueTransposedNode(node);
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
node.value
);
return tnode;
}
protected void processTransposedNode(
WritingTransposedNode tnode,
TransposedNode parentNode,
Link link,
int rowIndex
) {
if (parentNode != null) {
if (parentNode instanceof TransposedNodeWithChildren) {
TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
parentNode2.rowIndices.add(rowIndex);
parentNode2.children.add(tnode);
parentNode2.links.add(link);
}
} else {
addRootNode(tnode, rowIndex);
}
}
protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
JSONObject o = new JSONObject();
try {
if (value instanceof FreebaseTopic) {
FreebaseTopic topic = (FreebaseTopic) value;
o.put("id", topic.id);
o.put("name", topic.name);
} else {
o.put("v", value);
}
} catch (JSONException e) {
// ignore
}
connectJsonContext(parent, o, key);
return o;
}
protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
JSONObject o = new JSONObject();
connectJsonContext(parent, o, key);
try {
if (cell != null) {
o.put("v", cell.value);
if (cell.recon != null) {
o.put("recon", "rec" + cell.recon.id);
if (cell.recon.judgment == Judgment.Matched) {
o.put("id", cell.recon.match.id);
o.put("name", cell.recon.match.name);
}
// qa:display_context
{
StringBuffer sb2 = new StringBuffer();
sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
sb2.append(Long.toString(cell.recon.id));
sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
sb2.append(Long.toString(contextID));
sb2.append("\", \"meta\" : { \"row\" : ");
sb2.append(Integer.toString(rowIndex));
sb2.append(" } }");
writeLine(sb2.toString());
}
}
}
} catch (JSONException e) {
// ignore
}
return o;
}
protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
try {
if (parent == null) {
contextTreeRoot = o;
} else {
JSONArray a = null;
if (parent.has(key)) {
a = parent.getJSONArray(key);
} else {
a = new JSONArray();
parent.put(key, a);
}
a.put(o);
}
} catch (JSONException e) {
// ignore
}
}
protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
if (lastRootNode != null) {
lastRootNode.write(null, null, project, -1, -1, null);
writeContextTreeNode();
}
lastRootNode = tnode;
contextTreeRoot = null;
contextRowIndex = rowIndex;
contextRefCount = 0;
contextID++;
}
protected void writeContextTreeNode() {
if (contextTreeRoot != null && contextRefCount > 0) {
StringBuffer sb = new StringBuffer();
sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
sb.append(Long.toString(contextID));
sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
sb.append(Integer.toString(contextRowIndex));
sb.append(", \"data\" : ");
sb.append(contextTreeRoot.toString());
sb.append(" } }");
writeLine(sb.toString());
}
}
static protected Object validateValue(Object value, String valueType) {
if ("/type/datetime".equals(valueType)) {
if (value instanceof Calendar || value instanceof Date) {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
value = formatter.format(value instanceof Date ? ((Date) value)
: ((Calendar) value).getTime());
} else if (!(value instanceof String)) {
value = value.toString();
}
} else if ("/type/boolean".equals(valueType)) {
if (!(value instanceof Boolean)) {
value = Boolean.parseBoolean(value.toString());
}
} else if ("/type/int".equals(valueType)) {
if (value instanceof Number) {
value = ((Number) value).longValue();
} else {
try {
value = Long.parseLong(value.toString());
} catch (NumberFormatException e) {
value = null;
}
}
} else if ("/type/float".equals(valueType)) {
if (value instanceof Number) {
value = ((Number) value).floatValue();
} else {
try {
value = Float.parseFloat(value.toString());
} catch (NumberFormatException e) {
value = null;
}
}
} else if ("/type/double".equals(valueType)) {
if (value instanceof Number) {
value = ((Number) value).doubleValue();
} else {
try {
value = Double.parseDouble(value.toString());
} catch (NumberFormatException e) {
value = null;
}
}
}
return value;
}
}

453
extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
View File

@ -1,453 +0,0 @@
/*
Copyright 2010,2013 Google Inc. and contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.util;
import java.io.Serializable;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.model.ReconCandidate;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
public class FreebaseDataExtensionJob {
static public class DataExtension {
final public Object[][] data;
public DataExtension(Object[][] data) {
this.data = data;
}
}
static public class ColumnInfo {
final public List<String> names;
final public List<String> path;
final public FreebaseType expectedType;
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names;
this.path = path;
this.expectedType = expectedType;
}
}
final public JSONObject extension;
final public int columnCount;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
this.extension = obj;
this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ?
countColumns(obj.getJSONArray("properties"), columns, new ArrayList<String>(), new ArrayList<String>()) : 0;
}
public Map<String, FreebaseDataExtensionJob.DataExtension> extend(
Set<String> ids,
Map<String, ReconCandidate> reconCandidateMap
) throws Exception {
StringWriter writer = new StringWriter();
formulateQuery(ids, extension, writer);
String query = writer.toString();
String result = FreebaseUtils.mqlread(query);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result);
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
if (o.has("result")) {
JSONArray a = o.getJSONArray("result");
int l = a.length();
for (int i = 0; i < l; i++) {
JSONObject o2 = a.getJSONObject(i);
String id = o2.getString("id");
FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
if (ext != null) {
map.put(id, ext);
}
}
}
return map;
}
protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
List<Object[]> rows = new ArrayList<Object[]>();
collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
Object[][] data = new Object[rows.size()][columnCount];
rows.toArray(data);
return new DataExtension(data);
}
protected void storeKey(
List<Object[]> rows,
int row,
int col,
JSONObject key,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String keyval = key.getString("value");
while (row >= rows.size()) {
rows.add(new Object[columnCount]);
}
rows.get(row)[col] = keyval;
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
Object value,
Map<String, ReconCandidate> reconCandidateMap
) {
while (row >= rows.size()) {
rows.add(new Object[columnCount]);
}
rows.get(row)[col] = value;
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String id = obj.getString("id");
ReconCandidate rc;
if (reconCandidateMap.containsKey(id)) {
rc = reconCandidateMap.get(id);
} else {
rc = new ReconCandidate(
obj.getString("id"),
obj.getString("name"),
JSONUtilities.getStringArray(obj, "type"),
100
);
reconCandidateMap.put(id, rc);
}
storeCell(rows, row, col, rc, reconCandidateMap);
}
protected int[] collectResult(
List<Object[]> rows,
JSONObject extNode,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String propertyID = extNode.getString("id");
String expectedTypeID = extNode.getJSONObject("expected").getString("id");
JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ?
resultNode.getJSONArray(propertyID) : null;
if ("/type/key".equals(expectedTypeID)) {
if (a != null) {
int l = a.length();
for (int r = 0; r < l; r++) {
Object o = a.isNull(r) ? null : a.get(r);
if (o instanceof JSONObject) {
storeKey(rows, startRowIndex++, startColumnIndex, (JSONObject) o, reconCandidateMap);
}
}
}
// note that we still take up a column even if we don't have any data
return new int[] { startRowIndex, startColumnIndex + 1 };
} else if (expectedTypeID.startsWith("/type/")) {
if (a != null) {
int l = a.length();
for (int r = 0; r < l; r++) {
Object o = a.isNull(r) ? null : a.get(r);
if (o instanceof Serializable) {
storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap);
}
}
}
// note that we still take up a column even if we don't have any data
return new int[] { startRowIndex, startColumnIndex + 1 };
} else {
boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
if (a != null && a.length() > 0) {
int maxColIndex = startColumnIndex;
int l = a.length();
for (int r = 0; r < l; r++) {
Object v = a.isNull(r) ? null : a.get(r);
JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null;
int startColumnIndex2 = startColumnIndex;
int startRowIndex2 = startRowIndex;
if (isOwnColumn) {
if (o != null) {
storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap);
} else {
storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap);
}
}
if (hasSubProperties && o != null) {
int[] rowcol = collectResult(
rows,
extNode.getJSONArray("properties"),
o,
startRowIndex,
startColumnIndex2,
reconCandidateMap
);
startRowIndex2 = rowcol[0];
startColumnIndex2 = rowcol[1];
}
startRowIndex = startRowIndex2;
maxColIndex = Math.max(maxColIndex, startColumnIndex2);
}
return new int[] { startRowIndex, maxColIndex };
} else {
return new int[] {
startRowIndex,
startColumnIndex + countColumns(extNode, null, new ArrayList<String>(), new ArrayList<String>())
};
}
}
}
protected int[] collectResult(
List<Object[]> rows,
JSONArray subProperties,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
int maxStartRowIndex = startRowIndex;
int k = subProperties.length();
for (int c = 0; c < k; c++) {
int[] rowcol = collectResult(
rows,
subProperties.getJSONObject(c),
resultNode,
startRowIndex,
startColumnIndex,
reconCandidateMap
);
maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]);
startColumnIndex = rowcol[1];
}
return new int[] { maxStartRowIndex, startColumnIndex };
}
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("id|=");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
jsonWriter.endObject();
jsonWriter.endArray();
}
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
String propertyID = node.getString("id");
String expectedTypeID = node.getJSONObject("expected").getString("id");
writer.key(propertyID);
writer.array();
{
if (!expectedTypeID.startsWith("/type/") // not literal
|| "/type/key".equals(expectedTypeID)) {
writer.object();
writer.key("optional"); writer.value(true);
boolean hasLimit = false;
if (node.has("constraints") && !node.isNull("constraints")) {
JSONObject constraints = node.getJSONObject("constraints");
String[] names = JSONObject.getNames(constraints);
for (String name : names) {
Object value = constraints.get(name);
if (name.equals("limit")) {
hasLimit = true;
}
if (!name.contains(":") &&
!name.equals("limit") &&
!name.equals("optional") &&
!name.equals("count") &&
!name.equals("estimate-count") &&
!name.equals("sort") &&
!name.equals("return")) {
if (name.startsWith("!")) {
name = "!c:" + name.substring(1);
} else {
name = "c:" + name;
}
}
if (name.equals("sort")) {
String sortKey = (String) value;
if (sortKey.startsWith("-")) {
sortKey = sortKey.substring(1);
}
writer.key(sortKey); writer.value(null);
writer.key(name); writer.value(value);
} else {
writer.key(name); writer.value(value);
}
}
}
if (!hasLimit) {
writer.key("limit"); writer.value(10);
}
{
boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
if ("/type/key".equals(expectedTypeID)) {
writer.key("value"); writer.value(null);
} else {
writer.key("name"); writer.value(null);
writer.key("id"); writer.value(null);
}
writer.key("type"); writer.array(); writer.endArray();
}
if (hasSubProperties) {
formulateQueryNode(node.getJSONArray("properties"), writer);
}
}
writer.endObject();
}
}
writer.endArray();
}
static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
int l = propertiesA.length();
for (int i = 0; i < l; i++) {
formulateQueryNode(propertiesA.getJSONObject(i), writer);
}
}
static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
String name = obj.getString("name");
List<String> names2 = null;
List<String> path2 = null;
if (columns != null) {
names2 = new ArrayList<String>(names);
names2.add(name);
path2 = new ArrayList<String>(path);
path2.add(obj.getString("id"));
}
if (obj.has("properties") && !obj.isNull("properties")) {
boolean included = (obj.has("included") && obj.getBoolean("included"));
if (included && columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return (included ? 1 : 0) +
countColumns(obj.getJSONArray("properties"), columns, names2, path2);
} else {
if (columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return 1;
}
}
static protected int countColumns(JSONArray a, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
int c = 0;
int l = a.length();
for (int i = 0; i < l; i++) {
c += countColumns(a.getJSONObject(i), columns, names, path);
}
return c;
}
}

543
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
View File

@ -1,543 +0,0 @@
/*
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.freebase.util;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.exception.OAuthCommunicationException;
import oauth.signpost.exception.OAuthExpectationFailedException;
import oauth.signpost.exception.OAuthMessageSignerException;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.api.client.googleapis.batch.BatchRequest;
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.freebase.Freebase;
import com.google.api.services.freebase.FreebaseRequestInitializer;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
public class FreebaseUtils {
private static final String FREEBASE_API_VERSION = "v1";
// private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox";
private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc";
private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION;
private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch";
static final Logger logger = LoggerFactory.getLogger("freebase");
static final public String FREEBASE_HOST = "freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
static final private String AGENT_ID = "/en/google_refine";
static final private int SAMPLE_SIZE = 300;
static final private int JUDGES = 4;
public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
public static String getApiKey() {
PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
String key = (String) ps.get("freebase.api.key");
if (key == null) {
key = System.getProperty("refine.google_api_key");
}
return key == null ? API_KEY : key;
}
private static String getUserInfoURL(String host) {
// TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
return "http://api." + host + "/api/service/user_info";
}
private static String getMQLWriteURL(String host) {
return "http://api." + host + "/api/service/mqlwrite";
}
private static String getMQLReadURL(String host) {
return "http://api." + host + "/api/service/mqlread";
}
private static String getUserAgent() {
return RefineServlet.FULLNAME;
}
public static String getUserInfo(Credentials credentials, Provider provider)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// sign the request with the oauth library
consumer.sign(httpRequest);
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
public static String getUserBadges(Provider provider, String user_id)
throws ClientProtocolException, IOException, JSONException {
String query = "{" +
"'id' : '" + user_id + "'," +
"'!/type/usergroup/member' : [{" +
"'id' : null," +
"'key' : [{" +
"'namespace' : null" +
"}]" +
"}]" +
"}".replace("'", "\"");
return mqlread(provider, query);
}
/**
* Perform an MQLREAD operation using the credentials of the given OAuth provider
*
* @deprecated This will go away when we switch to Google authentication.
*/
@Deprecated
public static String mqlread(Provider provider, String query)
throws ClientProtocolException, IOException, JSONException {
JSONObject envelope = new JSONObject();
envelope.put("query", new JSONObject(query));
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("query", envelope.toString()));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
/**
* Perform a single unauthenticated MQLread.
*
* (wrapper method for a bunch of alternative implementations)
*/
static public String mqlread(String query)
throws IOException, JSONException {
// A bunch of implementations which don't work for MQLread, but do for other methods
// String result = rpcCall(query);
// String result = googleCall(query);
// String result = batchCall1(query);
String result = mqlreadBatchMime(query);
return result;
}
public static String mqlwrite(Credentials credentials, Provider provider, String query)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
JSONObject envelope = new JSONObject();
envelope.put("query", new JSONObject(query));
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("query", envelope.toString()));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// sign the request with the oauth library
consumer.sign(httpRequest);
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
private static String getTweezersParams(int sample_size, int judges) {
String o = "{" +
"'sample_size':" + sample_size + "," +
"'votes':{" +
"'reconciled':" + judges + "," +
"'invalid':" + judges + "," +
"'new':" + judges + "," +
"'skip':" + (judges + 2) +
"}" +
"}";
return o.replace('\'', '"');
}
public static String uploadTriples(
HttpServletRequest request,
String qa,
String source_name,
String source_id,
String mdo_id,
String triples
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
JSONObject mdo_info = new JSONObject();
mdo_info.put("name", source_name);
if (source_id != null) {
mdo_info.put("info_source",source_id);
}
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
if (user_info.has("username")) {
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("check_params", "false"));
if (mdo_id != null) {
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
}
if (Boolean.parseBoolean(qa)) {
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
}
String freeqKey = System.getProperty("freeq.key");
if (freeqKey != null) {
logger.warn("Found Freeq key, will bypass OAuth signature");
formparams.add(new BasicNameValuePair("apikey", freeqKey));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
if (freeqKey == null) {
logger.warn("Calculating OAuth signature");
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
// Is there something we can do to generate an oauth signature without consuming so much memory?
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
}
}
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
} else {
throw new RuntimeException("Invalid credentials");
}
}
static public String getFreeQUrl() {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL;
}
static final String BOUNDARY = "---theOpenRefineBoundary--=";
/**
* A hand rolled MIME multipart/mixed implementation for Google's Batch API
*/
static private String mqlreadBatchMime(String query) throws JSONException, IOException {
URL url = new URL(GOOGLE_BATCH_URL);
String service_url = FREEBASE_SERVICE_URL+"/mqlread";
// We could use the javax.mail package, but it's actually more trouble than it's worth
String body = "--" + BOUNDARY + "\n"
+ queryToMimeBodyPart("0", query, service_url, FreebaseUtils.getApiKey())
+ "\n--" + BOUNDARY + "\n" ;
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY);
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
Writer writer = new OutputStreamWriter(connection.getOutputStream());
try {
writer.write(body);
} finally {
writer.flush();
writer.close();
}
connection.connect();
String result = null;
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LoggerFactory.getLogger("freebase").error(
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+ errorStream);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
String boundary = s.substring(0,s.indexOf("\n"));
boundary = boundary.split("\r")[0];
String[] part = s.split(boundary); // part 0 is empty because of leading boundary
String[] sections = part[1].split("\r\n\r\n");
// Mime headers, followed by HTTP headers, followd by actual response
result = sections[2];
} finally {
is.close();
}
}
return result;
}
static String queryToMimeBodyPart(String query_name,
String query, String service_url, String api_key)
throws IOException {
// We could use the javax.mail package, but it's actually more trouble than it's worth
StringBuilder sb = new StringBuilder();
sb.append("Content-Type: application/http\n");
sb.append("Content-Transfer-Encoding: binary\n");
sb.append("Content-ID: " + query_name + "\n");
sb.append("\n");
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("query",query));
params.add(new BasicNameValuePair("key", api_key));
UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
sb.append(body);
sb.append("\n");
return sb.toString();
}
//////////////////////// Unused methods for future use /////////////////////
/**
* This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
*
* NOTE: JSONRPC has been deprecated and replaced by HTTP Batch (which also
* doesn't support MQLread, so perhaps we should just remove this))
*/
@SuppressWarnings("unused")
static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {
URL url = new URL(GOOGLE_RPC_URL);
JSONObject params = new JSONObject();
params.put("query",query);
params.put("key", FreebaseUtils.getApiKey());
JSONObject req1 = new JSONObject();
req1.put("jsonrpc","2.0");
req1.put("id","q0");
req1.put("method","freebase.mqlread");
req1.put("apiVersion", FREEBASE_API_VERSION);
req1.put("params",params);
JSONArray body = new JSONArray();
body.put(req1);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("Content-Type", "application/json"); //
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
try {
writer.write(body.toString());
} finally {
writer.flush();
writer.close();
}
connection.connect();
JSONArray result = null;
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LoggerFactory.getLogger("freebase").error(
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+ errorStream);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
result = ParsingUtilities.evaluateJsonStringToArray(s);
} finally {
is.close();
}
}
return result.getJSONObject(0);
}
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
private static final FreebaseRequestInitializer REQUEST_INITIALIZER =
new FreebaseRequestInitializer(FreebaseUtils.getApiKey());
/**
* Submit a single MQL read query via the standard Google client library
*/
@SuppressWarnings("unused")
static private String mqlreadFreebaseClient(String query)
throws IOException, JSONException {
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
.setApplicationName("OpenRefine")
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
.build();
InputStream is = client.mqlread(query).executeAsInputStream();
String result = ParsingUtilities.inputStreamToString(is);
return result;
}
/**
* Submit a single MQL query via the Batch endpoint
* (not supported by Google's Java client)
*/
@SuppressWarnings("unused")
static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException {
JSONObject response = null;
// FIXME: We really want JsonBatchCallback<Freebase> here, but it's not supported right now
JsonBatchCallback<Void> callback = new JsonBatchCallback<Void>() {
public void onSuccess(Void res, HttpHeaders responseHeaders) {
System.out.println(res);
}
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
System.out.println("Error Message: " + e.getMessage());
}
};
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
.setApplicationName("OpenRefine")
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
.build();
// FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
BatchRequest batch = client.batch();
client.mqlread(query).queue(batch, callback);
batch.execute();
return response;
}
}

25
extensions/jython/src/com/google/refine/jython/JythonEvaluable.java
View File

@ -33,29 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.jython;
import com.google.refine.expr.*;
import org.python.core.*;
import org.python.util.PythonInterpreter;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import org.python.core.Py;
import org.python.core.PyException;
import org.python.core.PyFloat;
import org.python.core.PyFunction;
import org.python.core.PyInteger;
import org.python.core.PyLong;
import org.python.core.PyNone;
import org.python.core.PyObject;
import org.python.core.PyString;
import org.python.util.PythonInterpreter;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable;
import com.google.refine.expr.HasFields;
import com.google.refine.expr.LanguageSpecificParser;
import com.google.refine.expr.ParsingException;
public class JythonEvaluable implements Evaluable {
static public LanguageSpecificParser createParser() {
@ -68,7 +55,7 @@ public class JythonEvaluable implements Evaluable {
};
}
private static final String s_functionName = "___temp___";
private final String s_functionName;
private static PythonInterpreter _engine;
@ -97,6 +84,8 @@ public class JythonEvaluable implements Evaluable {
}
public JythonEvaluable(String s) {
this.s_functionName = String.format("__temp_%d__", Math.abs(s.hashCode()));
// indent and create a function out of the code
String[] lines = s.split("\r\n|\r|\n");

47
extensions/jython/tests/com/google/refine/jython/JythonEvaluableTest.java Normal file
View File

@ -0,0 +1,47 @@
package com.google.refine.jython;
import com.google.refine.expr.CellTuple;
import com.google.refine.expr.Evaluable;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.util.Properties;
/**
* @author Maxim Galushka
*/
public class JythonEvaluableTest {
@Test
public void testJythonConcurrent(){
Properties props = new Properties();
Project project = new Project();
Row row = new Row(2);
row.setCell(0, new Cell("one", null));
row.setCell(0, new Cell("1", null));
props.put("columnName", "number");
props.put("true", "true");
props.put("false", "false");
props.put("rowIndex", "0");
props.put("value", 1);
props.put("project", project);
props.put("call", "number");
props.put("PI", "3.141592654");
props.put("cells", new CellTuple(project, row));
Evaluable eval1 = new JythonEvaluable("a = value\nreturn a * 2");
Long value1 = (Long) eval1.evaluate(props);
// create some unrelated evaluable
new JythonEvaluable("a = value\nreturn a * 10");
// repeat same previous test
Long value2 = (Long) eval1.evaluate(props);
Assert.assertEquals(value1, value2);
}
}

4
main/src/com/google/refine/browsing/Engine.java
View File

@ -91,7 +91,7 @@ public class Engine implements Jsonizable {
_mode = mode;
}
public FilteredRows getAllRows() {
public synchronized FilteredRows getAllRows() {
return new FilteredRows() {
@Override
public void accept(Project project, RowVisitor visitor) {
@ -134,7 +134,7 @@ public class Engine implements Jsonizable {
throw new InternalError("Unknown mode.");
}
public FilteredRecords getAllRecords() {
public synchronized FilteredRecords getAllRecords() {
return new FilteredRecords() {
@Override
public void accept(Project project, RecordVisitor visitor) {

7
main/src/com/google/refine/exporters/HtmlTableExporter.java
View File

@ -61,9 +61,12 @@ public class HtmlTableExporter implements WriterExporter {
public void startFile(JSONObject options) {
try {
writer.write("<html>\n");
writer.write("<head><title>");
writer.write("<head>\n");
writer.write("<title>");
writer.write(ProjectManager.singleton.getProjectMetadata(project.id).getName());
writer.write("</title></head>\n");
writer.write("</title>\n");
writer.write("<meta charset=\"utf-8\" />\n");
writer.write("</head>\n");
writer.write("<body>\n");
writer.write("<table>\n");

8
main/src/com/google/refine/expr/MetaParser.java
View File

@ -120,6 +120,14 @@ abstract public class MetaParser {
}, "value");
}
/**
* languagePrefix will be stored in the meta model as an identifier.
* so be careful when change it as it will break the backward compatibility for the old project
* @param languagePrefix
* @param name
* @param parser
* @param defaultExpression
*/
static public void registerLanguageParser(String languagePrefix, String name, LanguageSpecificParser parser, String defaultExpression) {
s_languages.put(languagePrefix, new LanguageInfo(name, parser, defaultExpression));
}

21
main/src/com/google/refine/importers/ExcelImporter.java
View File

@ -37,12 +37,14 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLException;
import org.apache.poi.common.usermodel.Hyperlink;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
@ -77,10 +79,7 @@ public class ExcelImporter extends TabularImportingParserBase {
public JSONObject createParserUIInitializationData(
ImportingJob job, List<JSONObject> fileRecords, String format) {
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
boolean xmlBased = "text/xml/xlsx".equals(format);
JSONUtilities.safePut(options, "xmlBased", xmlBased);
JSONArray sheetRecords = new JSONArray();
JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
try {
@ -88,8 +87,13 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file);
if (!is.markSupported()) {
is = new PushbackInputStream(is, 8);
}
try {
Workbook wb = xmlBased ?
Workbook wb = POIXMLDocument.hasOOXMLHeader(is) ?
new XSSFWorkbook(is) :
new HSSFWorkbook(new POIFSFileSystem(is));
@ -136,10 +140,13 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONObject options,
List<Exception> exceptions
) {
boolean xmlBased = JSONUtilities.getBoolean(options, "xmlBased", false);
Workbook wb = null;
if (!inputStream.markSupported()) {
inputStream = new PushbackInputStream(inputStream, 8);
}
try {
wb = xmlBased ?
wb = POIXMLDocument.hasOOXMLHeader(inputStream) ?
new XSSFWorkbook(inputStream) :
new HSSFWorkbook(new POIFSFileSystem(inputStream));
} catch (IOException e) {

16
main/src/com/google/refine/importers/MarcImporter.java
View File

@ -41,6 +41,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import org.json.JSONObject;
import org.marc4j.MarcException;
import org.marc4j.MarcPermissiveStreamReader;
import org.marc4j.MarcWriter;
import org.marc4j.MarcXmlWriter;
@ -58,14 +59,11 @@ public class MarcImporter extends XmlImporter {
@Override
public JSONObject createParserUIInitializationData(ImportingJob job, java.util.List<JSONObject> fileRecords, String format) {
if (fileRecords.size() > 0) {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
File tempFile = new File(file.getAbsolutePath()+".xml");
JSONUtilities.safePut(firstFileRecord, "location",
JSONUtilities.getString(firstFileRecord, "location", "")+".xml");
try {
InputStream inputStream = new FileInputStream(file);
@ -84,7 +82,14 @@ public class MarcImporter extends XmlImporter {
try {
outputStream.close();
inputStream.close();
file.delete(); // get rid of our original file
if (tempFile.length() == 0) // write failed. Most of time because of wrong Marc format
tempFile.delete();
else // only set json if write the temp file successfully:
JSONUtilities.safePut(firstFileRecord, "location",
JSONUtilities.getString(firstFileRecord, "location", "")+".xml");
// file.delete(); // get rid of our original file
} catch (IOException e) {
// Just ignore - not much we can do anyway
}
@ -93,11 +98,8 @@ public class MarcImporter extends XmlImporter {
logger.error("Failed to create temporary XML file from MARC file", e);
}
}
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
return options;
};
}

28
main/src/com/google/refine/importers/tree/ImportParameters.java Normal file
View File

@ -0,0 +1,28 @@
package com.google.refine.importers.tree;
public class ImportParameters {
boolean trimStrings;
boolean storeEmptyStrings;
boolean guessDataType;
boolean includeFileSources;
String fileSource;
public ImportParameters(boolean trimStrings, boolean storeEmptyStrings, boolean guessCellValueTypes,
boolean includeFileSources, String fileSource) {
this.trimStrings = trimStrings;
this.storeEmptyStrings = storeEmptyStrings;
this.guessDataType = guessCellValueTypes;
this.includeFileSources = includeFileSources;
this.fileSource = fileSource;
}
public ImportParameters(boolean trimStrings, boolean storeEmptyStrings, boolean guessCellValueTypes) {
this.trimStrings = trimStrings;
this.storeEmptyStrings = storeEmptyStrings;
this.guessDataType = guessCellValueTypes;
this.includeFileSources = false;
this.fileSource = "";
}
}

24
main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
View File

@ -48,7 +48,11 @@ import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress;
import com.google.refine.importers.ImportingParserBase;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingUtilities;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.util.JSONUtilities;
/**
@ -210,8 +214,22 @@ abstract public class TreeImportingParserBase extends ImportingParserBase {
boolean trimStrings = JSONUtilities.getBoolean(options, "trimStrings", true);
boolean storeEmptyStrings = JSONUtilities.getBoolean(options, "storeEmptyStrings", false);
boolean guessCellValueTypes = JSONUtilities.getBoolean(options, "guessCellValueTypes", true);
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2, trimStrings,
storeEmptyStrings,guessCellValueTypes);
// copied from TabularImportingParserBase
boolean includeFileSources = JSONUtilities.getBoolean(options, "includeFileSources", false);
String fileNameColumnName = "File";
if (includeFileSources) {
if (project.columnModel.getColumnByName(fileNameColumnName) == null) {
try {
project.columnModel.addColumn(
0, new Column(project.columnModel.allocateNewCellIndex(), fileNameColumnName), false);
} catch (ModelException e) {
// Ignore: We already checked for duplicate name.
}
}
}
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2,
new ImportParameters(trimStrings, storeEmptyStrings,guessCellValueTypes, includeFileSources,fileSource));
}
}

133
main/src/com/google/refine/importers/tree/XmlImportUtilities.java
View File

@ -245,26 +245,13 @@ public class XmlImportUtilities extends TreeImportUtilities {
return null;
}
@Deprecated
static public void importTreeData(
TreeReader parser,
Project project,
String[] recordPath,
ImportColumnGroup rootColumnGroup,
int limit
) {
importTreeData(parser, project, recordPath, rootColumnGroup, limit,true,false,true);
}
static public void importTreeData(
TreeReader parser,
Project project,
String[] recordPath,
ImportColumnGroup rootColumnGroup,
int limit,
boolean trimStrings,
boolean storeEmptyStrings,
boolean guessDataType
ImportParameters parameters
) {
if (logger.isTraceEnabled()) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)");
@ -273,7 +260,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
while (parser.hasNext()) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--,trimStrings,storeEmptyStrings,guessDataType);
findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--,parameters);
}
}
} catch (TreeReaderException e) {
@ -282,18 +269,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
@Deprecated
static protected void findRecord(
Project project,
TreeReader parser,
String[] recordPath,
int pathIndex,
ImportColumnGroup rootColumnGroup,
int limit
) throws TreeReaderException {
findRecord(project, parser, recordPath, pathIndex, rootColumnGroup, limit, true, false,true);
}
/**
*
* @param project
@ -310,9 +285,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
int pathIndex,
ImportColumnGroup rootColumnGroup,
int limit,
boolean trimStrings,
boolean storeEmptyStrings,
boolean guessDataType
ImportParameters parameters
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("findRecord(Project, TreeReader, String[], int, ImportColumnGroup - path:"+Arrays.toString(recordPath));
@ -331,7 +304,8 @@ public class XmlImportUtilities extends TreeImportUtilities {
while (parser.hasNext() && limit != 0) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--,trimStrings,storeEmptyStrings,guessDataType);
findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--,
parameters);
} else if (eventType == Token.EndEntity) {
break;
} else if (eventType == Token.Value) {
@ -340,13 +314,13 @@ public class XmlImportUtilities extends TreeImportUtilities {
String desiredFieldName = recordPath[pathIndex + 1];
String currentFieldName = parser.getFieldName();
if (desiredFieldName.equals(currentFieldName)) {
processFieldAsRecord(project, parser, rootColumnGroup,trimStrings,storeEmptyStrings,guessDataType);
processFieldAsRecord(project, parser, rootColumnGroup,parameters);
}
}
}
}
} else {
processRecord(project, parser, rootColumnGroup, trimStrings, storeEmptyStrings, guessDataType);
processRecord(project, parser, rootColumnGroup, parameters);
}
} else {
skip(parser);
@ -364,17 +338,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
/**
* @deprecated on 20120907 by tfmorris -use {@link #processRecord(Project, TreeReader, ImportColumnGroup, boolean, boolean, boolean)}
*/
@Deprecated
static protected void processRecord(
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup
) throws TreeReaderException {
processRecord(project, parser, rootColumnGroup, true, false, true);
}
/**
* processRecord parses Tree data for a single element and it's sub-elements,
@ -388,32 +351,18 @@ public class XmlImportUtilities extends TreeImportUtilities {
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup,
boolean trimStrings,
boolean storeEmptyStrings,
boolean guessDataType
ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processRecord(Project,TreeReader,ImportColumnGroup)");
}
ImportRecord record = new ImportRecord();
processSubRecord(project, parser, rootColumnGroup, record, 0, trimStrings, storeEmptyStrings, guessDataType);
addImportRecordToProject(record, project);
processSubRecord(project, parser, rootColumnGroup, record, 0, parameter);
addImportRecordToProject(record, project, parameter.includeFileSources, parameter.fileSource);
}
/**
* @deprecated 20120907 by tfmorris - use {@link #processFieldAsRecord(Project, TreeReader, ImportColumnGroup, boolean, boolean, boolean)}
*/
@Deprecated
static protected void processFieldAsRecord(
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup
) throws TreeReaderException {
processFieldAsRecord(project, parser, rootColumnGroup, true, false, true);
}
/**
* processFieldAsRecord parses Tree data for a single element and it's sub-elements,
* adding the parsed data as a row to the project
@ -426,9 +375,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup,
boolean trimStrings,
boolean storeEmptyStrings,
boolean guessDataType
ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processFieldAsRecord(Project,TreeReader,ImportColumnGroup)");
@ -437,10 +384,10 @@ public class XmlImportUtilities extends TreeImportUtilities {
ImportRecord record = null;
if (value instanceof String) {
String text = (String) value;
if (trimStrings) {
if (parameter.trimStrings) {
text = text.trim();
}
if (text.length() > 0 | !storeEmptyStrings) {
if (text.length() > 0 | !parameter.storeEmptyStrings) {
record = new ImportRecord();
addCell(
project,
@ -448,8 +395,8 @@ public class XmlImportUtilities extends TreeImportUtilities {
record,
parser.getFieldName(),
(String) value,
storeEmptyStrings,
guessDataType
parameter.storeEmptyStrings,
parameter.guessDataType
);
}
} else {
@ -463,20 +410,25 @@ public class XmlImportUtilities extends TreeImportUtilities {
);
}
if (record != null) {
addImportRecordToProject(record, project);
addImportRecordToProject(record, project,
parameter.includeFileSources, parameter.fileSource);
}
}
static protected void addImportRecordToProject(ImportRecord record, Project project) {
static protected void addImportRecordToProject(ImportRecord record, Project project,
boolean includeFileSources, String fileSource) {
for (List<Cell> row : record.rows) {
if (row.size() > 0) {
Row realRow = null;
Row realRow = new Row(row.size()); ;
for (int c = 0; c < row.size(); c++) {
if (c == 0 && includeFileSources) { // to add the file source:
realRow.setCell(
0,
new Cell(fileSource, null));
continue;
}
Cell cell = row.get(c);
if (cell != null) {
if (realRow == null) {
realRow = new Row(row.size());
}
realRow.setCell(c, cell);
}
}
@ -486,19 +438,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
}
/**
* @deprecated by tfmorris use {@link #processSubRecord(Project, TreeReader, ImportColumnGroup, ImportRecord, int, boolean, boolean, boolean)}
*/
@Deprecated
static protected void processSubRecord( Project project,
TreeReader parser,
ImportColumnGroup columnGroup,
ImportRecord record,
int level
) throws TreeReaderException {
processSubRecord(project, parser, columnGroup, record, level, true, false, true);
}
/**
*
@ -514,9 +453,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
ImportColumnGroup columnGroup,
ImportRecord record,
int level,
boolean trimStrings,
boolean storeEmptyStrings,
boolean guessDataType
ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processSubRecord(Project,TreeReader,ImportColumnGroup,ImportRecord) lvl:"+level+" "+columnGroup);
@ -536,18 +473,18 @@ public class XmlImportUtilities extends TreeImportUtilities {
int attributeCount = parser.getAttributeCount();
for (int i = 0; i < attributeCount; i++) {
String text = parser.getAttributeValue(i);
if (trimStrings) {
if (parameter.trimStrings) {
text = text.trim();
}
if (text.length() > 0 | !storeEmptyStrings) {
if (text.length() > 0 | !parameter.storeEmptyStrings) {
addCell(
project,
thisColumnGroup,
record,
composeName(parser.getAttributePrefix(i), parser.getAttributeLocalName(i)),
text,
storeEmptyStrings,
guessDataType
parameter.storeEmptyStrings,
parameter.guessDataType
);
}
}
@ -561,9 +498,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
thisColumnGroup,
record,
level+1,
trimStrings,
storeEmptyStrings,
guessDataType
parameter
);
} else if (//eventType == XMLStreamConstants.CDATA ||
eventType == Token.Value) { //XMLStreamConstants.CHARACTERS) {
@ -572,7 +507,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
if (value instanceof String) {
String text = (String) value;
addCell(project, thisColumnGroup, record, colName, text,
storeEmptyStrings, guessDataType);
parameter.storeEmptyStrings, parameter.guessDataType);
} else {
addCell(project, thisColumnGroup, record, colName, value);
}

12
main/tests/server/src/com/google/refine/tests/exporters/HtmlExporterTests.java
View File

@ -117,7 +117,9 @@ public class HtmlExporterTests extends RefineTest {
}
Assert.assertEquals(writer.toString(), "<html>\n" +
"<head><title>" + TEST_PROJECT_NAME + "</title></head>\n" +
"<head>\n" + "<title>" + TEST_PROJECT_NAME + "</title>\n" +
"<meta charset=\"utf-8\" />\n" +
"</head>\n" +
"<body>\n" +
"<table>\n" +
"<tr><th>column0</th><th>column1</th></tr>\n" +
@ -142,7 +144,9 @@ public class HtmlExporterTests extends RefineTest {
}
Assert.assertEquals(writer.toString(), "<html>\n" +
"<head><title>" + TEST_PROJECT_NAME + "</title></head>\n" +
"<head>\n" + "<title>" + TEST_PROJECT_NAME + "</title>\n" +
"<meta charset=\"utf-8\" />\n" +
"</head>\n" +
"<body>\n" +
"<table>\n" +
"<tr><td>row0cell0</td><td>row0cell1</td></tr>\n" +
@ -167,7 +171,9 @@ public class HtmlExporterTests extends RefineTest {
}
Assert.assertEquals(writer.toString(), "<html>\n" +
"<head><title>" + TEST_PROJECT_NAME + "</title></head>\n" +
"<head>\n" + "<title>" + TEST_PROJECT_NAME + "</title>\n" +
"<meta charset=\"utf-8\" />\n" +
"</head>\n" +
"<body>\n" +
"<table>\n" +
"<tr><th>column0</th><th>column1</th><th>column2</th></tr>\n" +

36
main/tests/server/src/com/google/refine/tests/importers/XmlImportUtilitiesStub.java
View File

@ -37,6 +37,7 @@ import java.io.Serializable;
import java.util.List;
import com.google.refine.importers.tree.ImportColumnGroup;
import com.google.refine.importers.tree.ImportParameters;
import com.google.refine.importers.tree.ImportRecord;
import com.google.refine.importers.tree.TreeReader;
import com.google.refine.importers.tree.XmlImportUtilities;
@ -49,42 +50,24 @@ public class XmlImportUtilitiesStub extends XmlImportUtilities {
}
public void ProcessSubRecordWrapper(Project project, TreeReader parser, ImportColumnGroup columnGroup,
ImportRecord record, int level,boolean trimStrings, boolean storeEmptyStrings, boolean guessDataType)
ImportRecord record, int level, ImportParameters parameter)
throws Exception {
super.processSubRecord(project, parser, columnGroup, record, level, trimStrings, storeEmptyStrings, guessDataType);
}
@Deprecated
public void ProcessSubRecordWrapper(Project project, TreeReader parser, ImportColumnGroup columnGroup,
ImportRecord record, int level)
throws Exception {
super.processSubRecord(project, parser, columnGroup, record, level, false, true, false);
super.processSubRecord(project, parser, columnGroup, record, level, parameter);
}
public void findRecordWrapper(Project project, TreeReader parser, String[] recordPath, int pathIndex,
ImportColumnGroup rootColumnGroup, boolean trimStrings, boolean storeEmptyStrings, boolean guessDataType)
throws Exception {
super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup, -1, trimStrings, storeEmptyStrings, guessDataType);
}
@Deprecated
public void findRecordWrapper(Project project, TreeReader parser, String[] recordPath, int pathIndex,
ImportColumnGroup rootColumnGroup)
throws Exception {
super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup, -1, true, false, true);
super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup, -1,
new ImportParameters(trimStrings, storeEmptyStrings, guessDataType));
}
public void processRecordWrapper(Project project, TreeReader parser, ImportColumnGroup rootColumnGroup,
boolean trimStrings, boolean storeEmptyStrings, boolean guessDataType)
throws Exception {
super.processRecord(project, parser, rootColumnGroup, trimStrings, storeEmptyStrings, guessDataType);
super.processRecord(project, parser, rootColumnGroup,
new ImportParameters(trimStrings, storeEmptyStrings, guessDataType));
}
@Deprecated
public void processRecordWrapper(Project project, TreeReader parser, ImportColumnGroup rootColumnGroup)
throws Exception {
super.processRecord(project, parser, rootColumnGroup, true, false, true);
}
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, Serializable value, int commonStartingRowIndex) {
super.addCell(project, columnGroup, record, columnLocalName, value);
@ -93,9 +76,4 @@ public class XmlImportUtilitiesStub extends XmlImportUtilities {
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex, boolean trimStrings, boolean storeEmptyStrings) {
super.addCell(project, columnGroup, record, columnLocalName, text, trimStrings, storeEmptyStrings);
}
@Deprecated
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex) {
super.addCell(project, columnGroup, record, columnLocalName, text, false, true);
}
}

21
main/tests/server/src/com/google/refine/tests/importers/XmlImportUtilitiesTests.java
View File

@ -53,6 +53,7 @@ import com.google.refine.importers.JsonImporter.JSONTreeReader;
import com.google.refine.importers.XmlImporter.XmlParser;
import com.google.refine.importers.tree.ImportColumn;
import com.google.refine.importers.tree.ImportColumnGroup;
import com.google.refine.importers.tree.ImportParameters;
import com.google.refine.importers.tree.ImportRecord;
import com.google.refine.importers.tree.TreeReader;
import com.google.refine.importers.tree.TreeReaderException;
@ -208,8 +209,8 @@ public class XmlImportUtilitiesTests extends RefineTest {
loadSampleXml();
String[] recordPath = new String[]{"library","book"};
XmlImportUtilitiesStub.importTreeData(createXmlParser(), project, recordPath, columnGroup, -1, false, true,
false);
XmlImportUtilitiesStub.importTreeData(createXmlParser(), project, recordPath, columnGroup, -1,
new ImportParameters(false, true, false));
log(project);
assertProjectCreated(project, 0, 6);
@ -229,8 +230,8 @@ public class XmlImportUtilitiesTests extends RefineTest {
loadData(XmlImporterTests.getSampleWithVaryingStructure());
String[] recordPath = new String[]{"library", "book"};
XmlImportUtilitiesStub.importTreeData(createXmlParser(), project, recordPath, columnGroup, -1, false, true,
false);
XmlImportUtilitiesStub.importTreeData(createXmlParser(), project, recordPath, columnGroup, -1,
new ImportParameters(false, true, false));
log(project);
assertProjectCreated(project, 0, 6);
@ -283,7 +284,8 @@ public class XmlImportUtilitiesTests extends RefineTest {
int pathIndex = 0;
try {
SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup);
SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup,
false, false, false);
} catch (Exception e) {
Assert.fail();
}
@ -302,7 +304,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
SUT.processRecordWrapper(project, parser, columnGroup, false, false, false);
} catch (Exception e) {
Assert.fail();
}
@ -323,7 +325,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
SUT.processRecordWrapper(project, parser, columnGroup, false, false, false);
} catch (Exception e) {
Assert.fail();
}
@ -348,7 +350,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
SUT.processRecordWrapper(project, parser, columnGroup, false, false, false);
} catch (Exception e) {
Assert.fail();
}
@ -372,7 +374,8 @@ public class XmlImportUtilitiesTests extends RefineTest {
ParserSkip();
try {
SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record,0);
SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record,0,
new ImportParameters(false, false, false));
} catch (Exception e) {
Assert.fail();
}

BIN
main/webapp/WEB-INF/lib/poi-3.12-20150511.jar Normal file
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/poi-3.8-20120326.jar
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/poi-ooxml-3.12-20150511.jar Normal file
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/poi-ooxml-3.8-20120326.jar
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.8-20120326.jar → main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.12-20150511.jar
View File

Binary file not shown.

22
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -204,14 +204,13 @@ function registerImporting() {
IM.registerFormat("text/rdf+n3", "RDF/N3 files", "RdfTriplesParserUI", new Packages.com.google.refine.importers.RdfTripleImporter());
IM.registerFormat("text/xml", "XML files", "XmlParserUI", new Packages.com.google.refine.importers.XmlImporter());
IM.registerFormat("text/xml/xlsx", "Excel (.xlsx) files", "ExcelParserUI", new Packages.com.google.refine.importers.ExcelImporter());
IM.registerFormat("binary/text/xml/xls/xlsx", "Excel files", "ExcelParserUI", new Packages.com.google.refine.importers.ExcelImporter());
IM.registerFormat("text/xml/ods", "Open Document Format spreadsheets (.ods)", "ExcelParserUI", new Packages.com.google.refine.importers.OdsImporter());
IM.registerFormat("text/xml/rdf", "RDF/XML files", "RdfTriplesParserUI", new Packages.com.google.refine.importers.RdfXmlTripleImporter());
IM.registerFormat("text/json", "JSON files", "JsonParserUI", new Packages.com.google.refine.importers.JsonImporter());
IM.registerFormat("text/marc", "MARC files", "XmlParserUI", new Packages.com.google.refine.importers.MarcImporter());
IM.registerFormat("binary", "Binary files"); // generic format, no parser to handle it
IM.registerFormat("binary/xls", "Excel files", "ExcelParserUI", new Packages.com.google.refine.importers.ExcelImporter());
IM.registerFormat("service", "Services"); // generic format, no parser to handle it
@ -228,8 +227,8 @@ function registerImporting() {
IM.registerExtension(".json", "text/json");
IM.registerExtension(".js", "text/json");
IM.registerExtension(".xls", "binary/xls");
IM.registerExtension(".xlsx", "text/xml/xlsx");
IM.registerExtension(".xls", "binary/text/xml/xls/xlsx");
IM.registerExtension(".xlsx", "binary/text/xml/xls/xlsx");
IM.registerExtension(".ods", "text/xml/ods");
@ -250,13 +249,13 @@ function registerImporting() {
IM.registerMimeType("text/rdf+n3", "text/rdf+n3");
IM.registerMimeType("application/msexcel", "binary/xls");
IM.registerMimeType("application/x-msexcel", "binary/xls");
IM.registerMimeType("application/x-ms-excel", "binary/xls");
IM.registerMimeType("application/vnd.ms-excel", "binary/xls");
IM.registerMimeType("application/x-excel", "binary/xls");
IM.registerMimeType("application/xls", "binary/xls");
IM.registerMimeType("application/x-xls", "text/xml/xlsx");
IM.registerMimeType("application/msexcel", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/x-msexcel", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/x-ms-excel", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/vnd.ms-excel", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/x-excel", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/xls", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/x-xls", "binary/text/xml/xls/xlsx");
IM.registerMimeType("application/vnd.oasis.opendocument.spreadsheet","text/xml/ods");
@ -429,7 +428,6 @@ function init() {
"scripts/reconciliation/recon-manager.js",
"scripts/reconciliation/recon-dialog.js",
"scripts/reconciliation/freebase-query-panel.js",
"scripts/reconciliation/standard-service-panel.js",
"scripts/dialogs/expression-preview-dialog.js",

1
main/webapp/modules/core/langs/translation-default.json
View File

@ -533,6 +533,7 @@
"specify-sep": "Please specify a separator.",
"warning-no-length": "No field length is specified.",
"warning-format": "The given field lengths are not properly formatted.",
"check-format": "Please check the file format.",
"split-into-col": "Split into several columns",
"add-based-col": "Add column based on this column",
"add-by-urls": "Add column by fetching URLs",

Some files were not shown because too many files have changed in this diff Show More