s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch master

Browse Source
This commit is contained in:
Tom Morris 2013-04-13 19:02:20 -04:00
commit 22a9acd04b
161 changed files with 12012 additions and 11600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.classpath
View File

@ -20,7 +20,7 @@
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/icu4j-4.2.1.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/icu4j-4.2.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.9.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
@ -46,7 +46,7 @@
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-api-1.5.6.jar"/> <classpathentry exported="true" kind="lib" path="server/lib/slf4j-api-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-log4j12-1.5.6.jar"/> <classpathentry exported="true" kind="lib" path="server/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar"/> <classpathentry exported="true" kind="lib" path="broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-2.5.3.jar"/> <classpathentry exported="true" kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.5.3.jar"/>
<classpathentry exported="true" kind="lib" path="broker/core/module/MOD-INF/lib/bdb-je-4.0.103.jar"/> <classpathentry exported="true" kind="lib" path="broker/core/module/MOD-INF/lib/bdb-je-4.0.103.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar"/> <classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar"/> <classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar"/>
@ -71,6 +71,11 @@
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.2-sources.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-core-1.2.1.2-sources.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-core-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/guava-13.0.jar"/> <classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/guava-13.0.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar" sourcepath="extensions/freebase/module/MOD-INF/libsrc/google-api-client-1.13.2-beta-sources.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar" sourcepath="extensions/freebase/module/MOD-INF/libsrc/google-api-services-freebase-v1-rev25-1.13.2-beta-sources.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/mail.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-client-1.13.2-beta-sources.jar"/> <classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-client-1.13.2-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v1-rev17-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-services-fusiontables-v1-rev17-1.13.2-beta-sources.jar"/> <classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v1-rev17-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-services-fusiontables-v1-rev17-1.13.2-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-http-client-1.13.1-beta-sources.jar"/> <classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-http-client-1.13.1-beta-sources.jar"/>

22
.gitattributes vendored Normal file
View File

@ -0,0 +1,22 @@
# Auto detect text files and perform LF normalization
* text=auto
# Custom for Visual Studio
*.cs diff=csharp
*.sln merge=union
*.csproj merge=union
*.vbproj merge=union
*.fsproj merge=union
*.dbproj merge=union
# Standard to msysgit
*.doc diff=astextplain
*.DOC diff=astextplain
*.docx diff=astextplain
*.DOCX diff=astextplain
*.dot diff=astextplain
*.DOT diff=astextplain
*.pdf diff=astextplain
*.PDF diff=astextplain
*.rtf diff=astextplain
*.RTF diff=astextplain

6
.project
View File

@ -10,8 +10,14 @@
<arguments> <arguments>
</arguments> </arguments>
</buildCommand> </buildCommand>
<buildCommand>
<name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec> </buildSpec>
<natures> <natures>
<nature>org.eclipse.jdt.core.javanature</nature> <nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
</natures> </natures>
</projectDescription> </projectDescription>

6
.settings/.jsdtscope Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry excluding="main/webapp/modules/core/MOD-INF/controller.js|main/webapp/modules/core/externals/|test-output/" kind="src" path=""/>
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.JRE_CONTAINER"/>
<classpathentry kind="output" path=""/>
</classpath>

63
.settings/org.eclipse.wst.jsdt.core.prefs Normal file
View File

@ -0,0 +1,63 @@
eclipse.preferences.version=1
org.eclipse.wst.jsdt.core.compiler.codegen.inlineJsrBytecode=disabled
org.eclipse.wst.jsdt.core.compiler.codegen.targetPlatform=1.2
org.eclipse.wst.jsdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.wst.jsdt.core.compiler.compliance=1.4
org.eclipse.wst.jsdt.core.compiler.debug.lineNumber=generate
org.eclipse.wst.jsdt.core.compiler.debug.localVariable=generate
org.eclipse.wst.jsdt.core.compiler.debug.sourceFile=generate
org.eclipse.wst.jsdt.core.compiler.doc.comment.support=enabled
org.eclipse.wst.jsdt.core.compiler.problem.assertIdentifier=warning
org.eclipse.wst.jsdt.core.compiler.problem.deprecation=warning
org.eclipse.wst.jsdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
org.eclipse.wst.jsdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
org.eclipse.wst.jsdt.core.compiler.problem.discouragedReference=warning
org.eclipse.wst.jsdt.core.compiler.problem.duplicateLocalVariables=warning
org.eclipse.wst.jsdt.core.compiler.problem.emptyStatement=ignore
org.eclipse.wst.jsdt.core.compiler.problem.fallthroughCase=ignore
org.eclipse.wst.jsdt.core.compiler.problem.fieldHiding=ignore
org.eclipse.wst.jsdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
org.eclipse.wst.jsdt.core.compiler.problem.forbiddenReference=error
org.eclipse.wst.jsdt.core.compiler.problem.hiddenCatchBlock=warning
org.eclipse.wst.jsdt.core.compiler.problem.indirectStaticAccess=ignore
org.eclipse.wst.jsdt.core.compiler.problem.invalidJavadoc=ignore
org.eclipse.wst.jsdt.core.compiler.problem.invalidJavadocTags=disabled
org.eclipse.wst.jsdt.core.compiler.problem.invalidJavadocTagsDeprecatedRef=disabled
org.eclipse.wst.jsdt.core.compiler.problem.invalidJavadocTagsNotVisibleRef=disabled
org.eclipse.wst.jsdt.core.compiler.problem.invalidJavadocTagsVisibility=default
org.eclipse.wst.jsdt.core.compiler.problem.localVariableHiding=ignore
org.eclipse.wst.jsdt.core.compiler.problem.looseVarDecleration=warning
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocComments=ignore
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocCommentsVisibility=default
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocTags=ignore
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocTagsOverriding=disabled
org.eclipse.wst.jsdt.core.compiler.problem.missingJavadocTagsVisibility=default
org.eclipse.wst.jsdt.core.compiler.problem.noEffectAssignment=warning
org.eclipse.wst.jsdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
org.eclipse.wst.jsdt.core.compiler.problem.nullReference=ignore
org.eclipse.wst.jsdt.core.compiler.problem.optionalSemicolon=warning
org.eclipse.wst.jsdt.core.compiler.problem.parameterAssignment=ignore
org.eclipse.wst.jsdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
org.eclipse.wst.jsdt.core.compiler.problem.potentialNullReference=ignore
org.eclipse.wst.jsdt.core.compiler.problem.redundantNullCheck=ignore
org.eclipse.wst.jsdt.core.compiler.problem.undefinedField=warning
org.eclipse.wst.jsdt.core.compiler.problem.undocumentedEmptyBlock=ignore
org.eclipse.wst.jsdt.core.compiler.problem.uninitializedGlobalVariable=ignore
org.eclipse.wst.jsdt.core.compiler.problem.uninitializedLocalVariable=warning
org.eclipse.wst.jsdt.core.compiler.problem.unnecessaryElse=ignore
org.eclipse.wst.jsdt.core.compiler.problem.unresolvedFieldReference=error
org.eclipse.wst.jsdt.core.compiler.problem.unresolvedMethodReference=error
org.eclipse.wst.jsdt.core.compiler.problem.unresolvedTypeReference=error
org.eclipse.wst.jsdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
org.eclipse.wst.jsdt.core.compiler.problem.unusedLabel=warning
org.eclipse.wst.jsdt.core.compiler.problem.unusedLocal=warning
org.eclipse.wst.jsdt.core.compiler.problem.unusedParameter=ignore
org.eclipse.wst.jsdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
org.eclipse.wst.jsdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
org.eclipse.wst.jsdt.core.compiler.problem.unusedPrivateMember=warning
org.eclipse.wst.jsdt.core.compiler.source=1.3
org.eclipse.wst.jsdt.core.compiler.taskCaseSensitive=enabled
org.eclipse.wst.jsdt.core.compiler.taskPriorities=NORMAL,HIGH,NORMAL
org.eclipse.wst.jsdt.core.compiler.taskTags=TODO,FIXME,XXX
semanticValidation=enabled

2
.settings/org.eclipse.wst.jsdt.ui.prefs Normal file
View File

@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.wst.jsdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates/>

1
.settings/org.eclipse.wst.jsdt.ui.superType.container Normal file
View File

@ -0,0 +1 @@
org.eclipse.wst.jsdt.launching.JRE_CONTAINER

1
.settings/org.eclipse.wst.jsdt.ui.superType.name Normal file
View File

@ -0,0 +1 @@
Global

3
broker/core/src/com/google/refine/broker/RefineBroker.java
View File

@ -92,7 +92,8 @@ public abstract class RefineBroker extends ButterflyModuleImpl {
static final protected Logger logger = LoggerFactory.getLogger("refine.broker"); static final protected Logger logger = LoggerFactory.getLogger("refine.broker");
static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info"; // TODO: This API is deprecated.
static final protected String USER_INFO_URL = "http://api.freebase.com/api/service/user_info";
static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials"; static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials";
static final protected String OAUTH_HEADER = "Authorization"; static final protected String OAUTH_HEADER = "Authorization";

74
build.xml
View File

@ -19,6 +19,8 @@
<property name="build.dir" value="build"/> <property name="build.dir" value="build"/>
<property name="dist.dir" value="dist"/> <property name="dist.dir" value="dist"/>
<property name="java_version" value="1.6"/>
<property name="appengine.app_id" value="openrefine-broker"/> <property name="appengine.app_id" value="openrefine-broker"/>
<property name="appengine.version" value="1"/> <property name="appengine.version" value="1"/>
<property name="appengine.sdk.dir" value="/opt/appengine"/> <property name="appengine.sdk.dir" value="/opt/appengine"/>
@ -119,7 +121,7 @@
<target name="build_server"> <target name="build_server">
<mkdir dir="${server.classes.dir}" /> <mkdir dir="${server.classes.dir}" />
<javac encoding="utf-8" srcdir="${server.src.dir}" destdir="${server.classes.dir}" debug="true" includeAntRuntime="no"> <javac source="${java_version}" target="${java_version}" encoding="utf-8" srcdir="${server.src.dir}" destdir="${server.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="server.class.path" /> <classpath refid="server.class.path" />
</javac> </javac>
<copy file="${server.src.dir}/log4j.properties" tofile="${server.classes.dir}/log4j.properties"/> <copy file="${server.src.dir}/log4j.properties" tofile="${server.classes.dir}/log4j.properties"/>
@ -127,7 +129,7 @@
<target name="build_webapp"> <target name="build_webapp">
<mkdir dir="${webapp.classes.dir}" /> <mkdir dir="${webapp.classes.dir}" />
<javac encoding="utf-8" destdir="${webapp.classes.dir}" debug="true" includeAntRuntime="no"> <javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${webapp.classes.dir}" debug="true" includeAntRuntime="no">
<src path="${webapp.src.dir}"/> <src path="${webapp.src.dir}"/>
<classpath refid="webapp.class.path" /> <classpath refid="webapp.class.path" />
</javac> </javac>
@ -136,7 +138,7 @@
<target name="build_tests" depends="build_webapp, build_server"> <target name="build_tests" depends="build_webapp, build_server">
<mkdir dir="${server.tests.classes.dir}" /> <mkdir dir="${server.tests.classes.dir}" />
<javac encoding="utf-8" srcdir="${server.tests.src.dir}" destdir="${server.tests.classes.dir}" debug="true" includeAntRuntime="no"> <javac source="${java_version}" target="${java_version}" encoding="utf-8" srcdir="${server.tests.src.dir}" destdir="${server.tests.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="tests.class.path" /> <classpath refid="tests.class.path" />
</javac> </javac>
<copy file="${server.tests.src.dir}/tests.log4j.properties" tofile="${server.tests.classes.dir}/tests.log4j.properties"/> <copy file="${server.tests.src.dir}/tests.log4j.properties" tofile="${server.tests.classes.dir}/tests.log4j.properties"/>
@ -150,7 +152,7 @@
<target name="build_broker_core"> <target name="build_broker_core">
<mkdir dir="${broker.core.dir}/module/MOD-INF/classes" /> <mkdir dir="${broker.core.dir}/module/MOD-INF/classes" />
<javac encoding="utf-8" destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no"> <javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
<src path="${broker.core.dir}/src/"/> <src path="${broker.core.dir}/src/"/>
<classpath refid="broker.class.path" /> <classpath refid="broker.class.path" />
</javac> </javac>
@ -158,7 +160,7 @@
<target name="build_broker_appengine" depends="build_broker_core"> <target name="build_broker_appengine" depends="build_broker_core">
<mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" /> <mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" />
<javac encoding="utf-8" destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no"> <javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
<src path="${broker.appengine.dir}/src/"/> <src path="${broker.appengine.dir}/src/"/>
<classpath refid="broker.appengine.class.path" /> <classpath refid="broker.appengine.class.path" />
</javac> </javac>
@ -217,40 +219,50 @@
</target> </target>
<target name="mac" depends="jar, prepare_webapp"> <target name="mac" depends="jar, prepare_webapp">
<taskdef
name="jarbundler"
classname="net.sourceforge.jarbundler.JarBundler"
classpath="${jarbundler.dir}/jarbundler-2.2.0.jar"
/>
<mkdir dir="${mac.dir}"/> <mkdir dir="${mac.dir}"/>
<jarbundler <taskdef name="bundleapp"
dir="${mac.dir}" classname="com.oracle.appbundler.AppBundlerTask"
name="OpenRefine" classpath="${appbundler.dir}/appbundler-1.0.jar" />
mainclass="com.google.refine.Refine"
icon="${graphics.dir}/icon/openrefine.icns" <bundleapp outputdirectory="${mac.dir}"
version="${version}" name="OpenRefine"
infostring="OpenRefine ${version}" displayname="OpenRefine ${version}"
icon="${graphics.dir}/icon/openrefine.icns"
identifier="org.openrefine.refine.Refine"
shortversion="${version}"
mainclassname="com/google/refine/Refine">
<runtime dir="${env.JAVA_HOME}"/>
<classpath dir="${build.dir}">
<include name="${fullname}-server.jar" />
</classpath>
<classpath dir="${server.lib.dir}">
<include name="**/*.jar" />
</classpath>
<option value="-Xms256M"/>
<option value="-Xmx1024M"/>
<option value="-Djava.awt.headless=true"/>
<option value="-Drefine.headless=true"/>
<option value="-Drefine.version=${revision}"/>
<option value="-Drefine.webapp=$APP_ROOT/Contents/Resource/${built.webapp.name}"/>
</bundleapp>
<copy todir="${mac.dir}/OpenRefine.app/Contents/Resource">
<fileset dir="${build.dir}" id="librarypathset" >
<include name="${built.webapp.name}/**/**" />
<exclude name="**/*.class" />
</fileset>
</copy>
<!--- Options from old jarbundler code not supported by appbundler
aboutmenuname="OpenRefine" aboutmenuname="OpenRefine"
workingdirectory="$APP_PACKAGE/Contents/Resources" workingdirectory="$APP_PACKAGE/Contents/Resources"
jvmversion="1.6+"
bundleid="com.google.refine.Refine"
vmoptions="-Xms256M -Xmx1024M -Drefine.version=${revision}"
antialiasedgraphics="true" antialiasedgraphics="true"
antialiasedtext="true" antialiasedtext="true"
liveresize="true" liveresize="true"
growboxintrudes="true" growboxintrudes="true"
screenmenu="true"> screenmenu="true">
<jarfilelist dir="${build.dir}" files="${fullname}-server.jar" /> -->
<jarfileset dir="${server.lib.dir}">
<exclude name="**/.svn" />
<include name="**/*.jar" />
</jarfileset>
<resourcefileset dir="${built.webapp.dir}/..">
<include name="${built.webapp.name}/**" />
</resourcefileset>
</jarbundler>
</target> </target>
<target name="windows" depends="jar, prepare_webapp"> <target name="windows" depends="jar, prepare_webapp">
@ -277,7 +289,7 @@
fileVersion="${full_version}" fileVersion="${full_version}"
txtFileVersion="${version}" txtFileVersion="${version}"
fileDescription="openrefine" fileDescription="openrefine"
copyright="Copyright (c) 2012 OpenRefine contributors, 2010, Google, Inc." copyright="Copyright (c) 2013 OpenRefine contributors, 2010, Google, Inc."
productVersion="${full_version}" productVersion="${full_version}"
txtProductVersion="${full_version}" txtProductVersion="${full_version}"
productName="OpenRefine" productName="OpenRefine"

BIN
extensions/freebase/module/MOD-INF/lib/commons-logging-1.1.1.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/google-oauth-client-servlet-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/lib/mail.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/libsrc/google-api-client-1.13.2-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/freebase/module/MOD-INF/libsrc/google-api-services-freebase-v1-rev25-1.13.2-beta-sources.jar Normal file
View File

Binary file not shown.

4
extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js
View File

@ -139,12 +139,12 @@ ExtendDataPreviewDialog.prototype._show = function(properties) {
} }
var suggestConfig = { var suggestConfig = {
filter: '(all type:/type/property)', type: '/type/property', // NOTE: requires patched Suggest to pass this through
// Default returns id, lang, mid, name, notable {id,name}, score // Default returns id, lang, mid, name, notable {id,name}, score
mql_output : JSON.stringify({'name':null,'id':null,'mid':null, '/type/property/expected_type':{'name':null,'id':null}}), mql_output : JSON.stringify({'name':null,'id':null,'mid':null, '/type/property/expected_type':{'name':null,'id':null}}),
}; };
if ((this._column.reconConfig) && (this._column.reconConfig.type)) { if ((this._column.reconConfig) && (this._column.reconConfig.type)) {
suggestConfig.filter = '(all type:/type/property (any namespace:/type/object namespace:' + this._column.reconConfig.type.id + '))'; suggestConfig.filter = '(should (any namespace:/type/object namespace:/common/topic namespace:' + this._column.reconConfig.type.id + '))';
} }
this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) { this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {

1
extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js
View File

@ -46,6 +46,7 @@ FreebaseLoadingDialog.prototype._createDialog = function() {
var authorization = this._elmts.authorization; var authorization = this._elmts.authorization;
var loadButton = this._elmts.loadButton; var loadButton = this._elmts.loadButton;
// TODO: Does this need to be redone for the new API / freebase.com site?
var check_authorization = function(cont) { var check_authorization = function(cont) {
var check_authorization_url = "command/freebase/check-authorization/" + provider; var check_authorization_url = "command/freebase/check-authorization/" + provider;
$.get(check_authorization_url, function(data) { $.get(check_authorization_url, function(data) {

2
extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js
View File

@ -99,6 +99,8 @@ SchemaAlignment._batchSearch = function(queries, onDone) {
batch[key] = queries[key]; batch[key] = queries[key];
} }
// TODO: New API doesn't accept multiple queries
// args.push("https://www.googleapis.com/freebase/v1/search?key=" + Freebase.API_KEY + "&" +
args.push("http://api.freebase.com/api/service/search?" + args.push("http://api.freebase.com/api/service/search?" +
$.param({ "queries" : JSON.stringify(batch) }) + "&callback=?"); $.param({ "queries" : JSON.stringify(batch) }) + "&callback=?");

6
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js
View File

@ -308,7 +308,7 @@ SchemaAlignmentDialog.UILink.prototype._showPropertySuggestPopup = function(elmt
}; };
var sourceTypeID = this._parentUINode.getExpectedType(); var sourceTypeID = this._parentUINode.getExpectedType();
if (sourceTypeID !== null) { if (sourceTypeID !== null) {
suggestOptions.filter = '(all type:/type/property (any namespace:/type/object namespace:' + sourceTypeID + '))' suggestOptions.filter = '(all type:/type/property (should namespace:' + sourceTypeID + '))'
} }
input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); }); input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
@ -340,15 +340,13 @@ SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
var dismissBusy = DialogSystem.showBusy(); var dismissBusy = DialogSystem.showBusy();
$.getJSON( $.getJSON(
"http://api.freebase.com/api/service/mqlread?query=" + JSON.stringify({ "https://www.googleapis.com/freebase/v1/mqlread?key="+ Freebase.API_KEY + "&query=" + JSON.stringify({
query: {
"id" : this._link.property.id, "id" : this._link.property.id,
"type" : "/type/property", "type" : "/type/property",
"expected_type" : { "expected_type" : {
"id" : null, "id" : null,
"name" : null, "name" : null,
"/freebase/type_hints/mediator" : null "/freebase/type_hints/mediator" : null
}
} }
}) + "&callback=?", }) + "&callback=?",
null, null,

10
extensions/freebase/module/scripts/util/freebase.js
View File

@ -33,12 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
var Freebase = {}; var Freebase = {};
Freebase.API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
Freebase.mqlread = function(query, options, onDone) { Freebase.mqlread = function(query, options, onDone) {
var params = {}; var params = {};
var queryEnv = {
"query": query
};
// TODO: Options need to be handled differently for new API - but this doesn't appear to be used
if (options) { if (options) {
for (var n in options) { for (var n in options) {
if (options.hasOwnProperty(n)) { if (options.hasOwnProperty(n)) {
@ -52,10 +52,10 @@ Freebase.mqlread = function(query, options, onDone) {
} }
} }
params.query = JSON.stringify(queryEnv); params.query = JSON.stringify(query);
$.getJSON( $.getJSON(
"http://api.freebase.com/api/service/mqlread?" + $.param(params) + "&callback=?", "https://www.googleapis.com/freebase/v1/mqlread?key=" + Freebase.API_KEY + $.param(params) + "&callback=?",
null, null,
onDone, onDone,
"jsonp" "jsonp"

6
extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java
View File

@ -44,6 +44,11 @@ import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider; import com.google.refine.oauth.Provider;
/**
* Perform an MQLread on the server, using the client's credentials.
*
* Currently unused. All client code calls the Freebase API directly.
*/
public class MQLReadCommand extends Command { public class MQLReadCommand extends Command {
@Override @Override
@ -55,6 +60,7 @@ public class MQLReadCommand extends Command {
response.setCharacterEncoding("UTF-8"); response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json"); response.setHeader("Content-Type", "application/json");
String query = request.getParameter("query"); String query = request.getParameter("query");
@SuppressWarnings("deprecation")
String result = FreebaseUtils.mqlread(provider,query); String result = FreebaseUtils.mqlread(provider,query);
response.getWriter().write(result); response.getWriter().write(result);
} catch (Exception e) { } catch (Exception e) {

145
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.InputStream; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -48,6 +46,7 @@ import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
@ -95,7 +94,7 @@ public class GuidBasedReconConfig extends StrictReconConfig {
@Override @Override
public int getBatchSize() { public int getBatchSize() {
return 10; return 50;
} }
@Override @Override
@ -119,87 +118,48 @@ public class GuidBasedReconConfig extends StrictReconConfig {
Map<String, Recon> guidToRecon = new HashMap<String, Recon>(); Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
try { try {
String query = null; String query = buildQuery(jobs);
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object(); String s = FreebaseUtils.mqlread(query);
jsonWriter.key("query"); JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null); if (o.has("result")) {
jsonWriter.key("name"); jsonWriter.value(null); JSONArray results = o.getJSONArray("result");
jsonWriter.key("guid"); jsonWriter.value(null); int count = results.length();
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("guid|="); for (int i = 0; i < count; i++) {
jsonWriter.array(); JSONObject result = results.getJSONObject(i);
for (ReconJob job : jobs) {
jsonWriter.value(((GuidBasedReconJob) job).guid);
}
jsonWriter.endArray();
jsonWriter.endObject(); String guid = result.getString("guid");
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); JSONArray types = result.getJSONArray("type");
} String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
StringBuffer sb = new StringBuffer(1024); typeIDs[j] = types.getString(j);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String guid = result.getString("guid");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
@ -213,4 +173,33 @@ public class GuidBasedReconConfig extends StrictReconConfig {
return recons; return recons;
} }
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("guid|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((GuidBasedReconJob) job).guid);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
return query;
}
} }

147
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.InputStream; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -48,6 +46,7 @@ import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
@ -99,7 +98,7 @@ public class IdBasedReconConfig extends StrictReconConfig {
@Override @Override
public int getBatchSize() { public int getBatchSize() {
return 10; return 40;
} }
@Override @Override
@ -123,87 +122,47 @@ public class IdBasedReconConfig extends StrictReconConfig {
Map<String, Recon> idToRecon = new HashMap<String, Recon>(); Map<String, Recon> idToRecon = new HashMap<String, Recon>();
try { try {
String query = null; String query = buildQuery(jobs);
{ String s = FreebaseUtils.mqlread(query);
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object(); JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
jsonWriter.key("query"); if (o.has("result")) {
jsonWriter.array(); JSONArray results = o.getJSONArray("result");
jsonWriter.object(); int count = results.length();
jsonWriter.key("id"); jsonWriter.value(null); for (int i = 0; i < count; i++) {
jsonWriter.key("name"); jsonWriter.value(null); JSONObject result = results.getJSONObject(i);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("id|="); String id = result.getString("id");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((IdBasedReconJob) job).id);
}
jsonWriter.endArray();
jsonWriter.endObject(); JSONArray types = result.getJSONArray("type");
jsonWriter.endArray(); String[] typeIDs = new String[types.length()];
jsonWriter.endObject(); for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String id = result.getString("id");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
@ -218,4 +177,34 @@ public class IdBasedReconConfig extends StrictReconConfig {
return recons; return recons;
} }
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("id|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((IdBasedReconJob) job).id);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
}
return query;
}
} }

171
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
View File

@ -33,10 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.InputStream; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -49,6 +47,7 @@ import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic; import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
@ -99,7 +98,7 @@ public class KeyBasedReconConfig extends StrictReconConfig {
@Override @Override
public int getBatchSize() { public int getBatchSize() {
return 10; return 40;
} }
@Override @Override
@ -124,100 +123,47 @@ public class KeyBasedReconConfig extends StrictReconConfig {
Map<String, Recon> keyToRecon = new HashMap<String, Recon>(); Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
try { try {
String query = null; String query = buildQuery(jobs);
{ String s = FreebaseUtils.mqlread(query);
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object(); JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
jsonWriter.key("query"); if (o.has("result")) {
jsonWriter.array(); JSONArray results = o.getJSONArray("result");
jsonWriter.object(); int count = results.length();
jsonWriter.key("id"); jsonWriter.value(null); for (int i = 0; i < count; i++) {
jsonWriter.key("name"); jsonWriter.value(null); JSONObject result = results.getJSONObject(i);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("key"); String key = result.getJSONArray("key").getJSONObject(0).getString("value");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("namespace"); JSONArray types = result.getJSONArray("type");
jsonWriter.object(); String[] typeIDs = new String[types.length()];
jsonWriter.key("id"); jsonWriter.value(namespace.id); for (int j = 0; j < typeIDs.length; j++) {
jsonWriter.endObject(); typeIDs[j] = types.getString(j);
jsonWriter.key("value"); jsonWriter.value(null);
jsonWriter.key("value|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((KeyBasedReconJob) job).key);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
@ -232,4 +178,47 @@ public class KeyBasedReconConfig extends StrictReconConfig {
return recons; return recons;
} }
private String buildQuery(List<ReconJob> jobs)
throws JSONException {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("key");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("namespace");
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(namespace.id);
jsonWriter.endObject();
jsonWriter.key("value"); jsonWriter.value(null);
jsonWriter.key("value|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((KeyBasedReconJob) job).key);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
query = stringWriter.toString();
}
return query;
}
} }

1
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -40,7 +40,6 @@ import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig; import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig { abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread";
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match"); String match = obj.getString("match");

2
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/SchemaHelper.java
View File

@ -13,7 +13,7 @@ import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
public class SchemaHelper { public class SchemaHelper {
static private final String helperURL = "http://2.refine-helper.stefanomazzocchi.user.dev.freebaseapps.com/"; static private final String helperURL = "http://openrefine-helper.freebaseapps.com/";
static private class PropertyInfo { static private class PropertyInfo {
String fromTypeID; String fromTypeID;

71
extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -31,19 +31,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
/**
*
*/
package com.google.refine.freebase.util; package com.google.refine.freebase.util;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable; import java.io.Serializable;
import java.io.StringWriter; import java.io.StringWriter;
import java.io.Writer; import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -97,35 +89,31 @@ public class FreebaseDataExtensionJob {
) throws Exception { ) throws Exception {
StringWriter writer = new StringWriter(); StringWriter writer = new StringWriter();
formulateQuery(ids, extension, writer); formulateQuery(ids, extension, writer);
String query = writer.toString(); String query = writer.toString();
InputStream is = doMqlRead(query);
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>(); String result = FreebaseUtils.mqlread(query);
if (o.has("result")) {
JSONArray a = o.getJSONArray("result");
int l = a.length();
for (int i = 0; i < l; i++) { JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result);
JSONObject o2 = a.getJSONObject(i); Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
String id = o2.getString("id"); if (o.has("result")) {
FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap); JSONArray a = o.getJSONArray("result");
int l = a.length();
if (ext != null) { for (int i = 0; i < l; i++) {
map.put(id, ext); JSONObject o2 = a.getJSONObject(i);
} String id = o2.getString("id");
FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
if (ext != null) {
map.put(id, ext);
} }
} }
return map;
} finally {
is.close();
} }
return map;
} }
protected FreebaseDataExtensionJob.DataExtension collectResult( protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj, JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap Map<String, ReconCandidate> reconCandidateMap
@ -312,34 +300,10 @@ public class FreebaseDataExtensionJob {
} }
static protected InputStream doMqlRead(String query) throws IOException {
URL url = new URL("http://api.freebase.com/api/service/mqlread");
URLConnection connection = url.openConnection();
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
try {
String body = "extended=1&query=" + ParsingUtilities.encode(query);
dos.writeBytes(body);
} finally {
dos.flush();
dos.close();
}
connection.connect();
return connection.getInputStream();
}
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException { static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer); JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -357,7 +321,6 @@ public class FreebaseDataExtensionJob {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
} }
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException { static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {

253
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -34,6 +34,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.util; package com.google.refine.freebase.util;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -56,19 +62,42 @@ import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair; import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreProtocolPNames; import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.api.client.googleapis.batch.BatchRequest;
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.freebase.Freebase;
import com.google.api.services.freebase.FreebaseRequestInitializer;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet; import com.google.refine.RefineServlet;
import com.google.refine.oauth.Credentials; import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider; import com.google.refine.oauth.Provider;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
public class FreebaseUtils { public class FreebaseUtils {
private static final String FREEBASE_API_VERSION = "v1";
// private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox";
private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc";
private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION;
private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch";
static final Logger logger = LoggerFactory.getLogger("freebase"); static final Logger logger = LoggerFactory.getLogger("freebase");
static final public String FREEBASE_HOST = "freebase.com"; static final public String FREEBASE_HOST = "freebase.com";
@ -80,7 +109,19 @@ public class FreebaseUtils {
static final private int SAMPLE_SIZE = 300; static final private int SAMPLE_SIZE = 300;
static final private int JUDGES = 4; static final private int JUDGES = 4;
public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
public static String getApiKey() {
PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
String key = (String) ps.get("freebase.api.key");
if (key == null) {
key = System.getProperty("refine.google_api_key");
}
return key == null ? API_KEY : key;
}
private static String getUserInfoURL(String host) { private static String getUserInfoURL(String host) {
// TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
return "http://api." + host + "/api/service/user_info"; return "http://api." + host + "/api/service/user_info";
} }
@ -134,6 +175,12 @@ public class FreebaseUtils {
return mqlread(provider, query); return mqlread(provider, query);
} }
/**
* Perform an MQLREAD operation using the credentials of the given OAuth provider
*
* @deprecated This will go away when we switch to Google authentication.
*/
@Deprecated
public static String mqlread(Provider provider, String query) public static String mqlread(Provider provider, String query)
throws ClientProtocolException, IOException, JSONException { throws ClientProtocolException, IOException, JSONException {
@ -159,6 +206,24 @@ public class FreebaseUtils {
return EntityUtils.toString(httpResponse.getEntity()); return EntityUtils.toString(httpResponse.getEntity());
} }
/**
* Perform a single unauthenticated MQLread.
*
* (wrapper method for a bunch of alternative implementations)
*/
static public String mqlread(String query)
throws IOException, JSONException {
// A bunch of implementations which don't work for MQLread, but do for other methods
// String result = rpcCall(query);
// String result = googleCall(query);
// String result = batchCall1(query);
String result = mqlreadBatchMime(query);
return result;
}
public static String mqlwrite(Credentials credentials, Provider provider, String query) public static String mqlwrite(Credentials credentials, Provider provider, String query)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
@ -286,4 +351,190 @@ public class FreebaseUtils {
return url != null ? url : FREEQ_URL; return url != null ? url : FREEQ_URL;
} }
static final String BOUNDARY = "---theOpenRefineBoundary--=";
/**
* A hand rolled MIME multipart/mixed implementation for Google's Batch API
*/
static private String mqlreadBatchMime(String query) throws JSONException, IOException {
URL url = new URL(GOOGLE_BATCH_URL);
String service_url = FREEBASE_SERVICE_URL+"/mqlread";
// We could use the javax.mail package, but it's actually more trouble than it's worth
String body = "--" + BOUNDARY + "\n"
+ queryToMimeBodyPart("0", query, service_url, FreebaseUtils.getApiKey())
+ "\n--" + BOUNDARY + "\n" ;
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY);
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
Writer writer = new OutputStreamWriter(connection.getOutputStream());
try {
writer.write(body);
} finally {
writer.flush();
writer.close();
}
connection.connect();
String result = null;
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LoggerFactory.getLogger("freebase").error(
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+ errorStream);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
String boundary = s.substring(0,s.indexOf("\n"));
boundary = boundary.split("\r")[0];
String[] part = s.split(boundary); // part 0 is empty because of leading boundary
String[] sections = part[1].split("\r\n\r\n");
// Mime headers, followed by HTTP headers, followd by actual response
result = sections[2];
} finally {
is.close();
}
}
return result;
}
static String queryToMimeBodyPart(String query_name,
String query, String service_url, String api_key)
throws IOException {
// We could use the javax.mail package, but it's actually more trouble than it's worth
StringBuilder sb = new StringBuilder();
sb.append("Content-Type: application/http\n");
sb.append("Content-Transfer-Encoding: binary\n");
sb.append("Content-ID: " + query_name + "\n");
sb.append("\n");
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("query",query));
params.add(new BasicNameValuePair("key", api_key));
UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
sb.append(body);
sb.append("\n");
return sb.toString();
}
//////////////////////// Unused methods for future use /////////////////////
/**
* This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
*/
@SuppressWarnings("unused")
static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {
URL url = new URL(GOOGLE_RPC_URL);
JSONObject params = new JSONObject();
params.put("query",query);
params.put("key", FreebaseUtils.getApiKey());
JSONObject req1 = new JSONObject();
req1.put("jsonrpc","2.0");
req1.put("id","q0");
req1.put("method","freebase.mqlread");
req1.put("apiVersion", FREEBASE_API_VERSION);
req1.put("params",params);
JSONArray body = new JSONArray();
body.put(req1);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("Content-Type", "application/json"); //
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
try {
writer.write(body.toString());
} finally {
writer.flush();
writer.close();
}
connection.connect();
JSONArray result = null;
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LoggerFactory.getLogger("freebase").error(
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+ errorStream);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
result = ParsingUtilities.evaluateJsonStringToArray(s);
} finally {
is.close();
}
}
return result.getJSONObject(0);
}
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
private static final FreebaseRequestInitializer REQUEST_INITIALIZER =
new FreebaseRequestInitializer(FreebaseUtils.getApiKey());
/**
* Submit a single MQL read query via the standard Google client library
*/
@SuppressWarnings("unused")
static private String mqlreadFreebaseClient(String query)
throws IOException, JSONException {
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
.setApplicationName("OpenRefine")
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
.build();
InputStream is = client.mqlread(query).executeAsInputStream();
String result = ParsingUtilities.inputStreamToString(is);
return result;
}
/**
* Submit a single MQL query via the Batch endpoint
* (not supported by Google's Java client)
*/
@SuppressWarnings("unused")
static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException {
JSONObject response = null;
// FIXME: We really want JsonBatchCallback<Freebase> here, but it's not supported right now
JsonBatchCallback<Void> callback = new JsonBatchCallback<Void>() {
public void onSuccess(Void res, HttpHeaders responseHeaders) {
System.out.println(res);
}
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
System.out.println("Error Message: " + e.getMessage());
}
};
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
.setApplicationName("OpenRefine")
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
.build();
// FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
BatchRequest batch = client.batch();
client.mqlread(query).queue(batch, callback);
batch.execute();
return response;
}
} }

BIN
extensions/jython/module/MOD-INF/lib/jython-2.5.3.jar → extensions/jython/module/MOD-INF/lib/jython-standalone-2.5.3.jar
View File

Binary file not shown.

2
extensions/jython/src/com/google/refine/jython/JythonEvaluable.java
View File

@ -128,7 +128,7 @@ public class JythonEvaluable implements Evaluable {
return unwrap(result); return unwrap(result);
} catch (PyException e) { } catch (PyException e) {
return new EvalError(e.getMessage()); return new EvalError(e.toString());
} }
} }

26
main/src/com/google/refine/InterProjectModel.java
View File

@ -99,27 +99,33 @@ public class InterProjectModel {
computeJoin(join); computeJoin(join);
_joins.put(key, join); synchronized (_joins) {
_joins.put(key, join);
}
} }
return _joins.get(key); return _joins.get(key);
} }
public void flushJoinsInvolvingProject(long projectID) { public void flushJoinsInvolvingProject(long projectID) {
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) { synchronized (_joins) {
ProjectJoin join = entry.getValue(); for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
if (join.fromProjectID == projectID || join.toProjectID == projectID) { ProjectJoin join = entry.getValue();
_joins.remove(entry.getKey()); if (join.fromProjectID == projectID || join.toProjectID == projectID) {
_joins.remove(entry.getKey());
}
} }
} }
} }
public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) { public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) {
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) { synchronized (_joins) {
ProjectJoin join = entry.getValue(); for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) || ProjectJoin join = entry.getValue();
join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) { if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) ||
_joins.remove(entry.getKey()); join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) {
_joins.remove(entry.getKey());
}
} }
} }
} }

2
main/src/com/google/refine/browsing/facets/TimeRangeFacet.java
View File

@ -157,7 +157,7 @@ public class TimeRangeFacet extends RangeFacet {
@Override @Override
protected boolean checkValue(long t) { protected boolean checkValue(long t) {
return t >= _from && t < _to; return t >= _from && t <= _to;
}; };
}; };
} else { } else {

3
main/src/com/google/refine/exporters/CsvExporter.java
View File

@ -81,6 +81,7 @@ public class CsvExporter implements WriterExporter{
JSONUtilities.getString(options, "separator", Character.toString(this.separator)); JSONUtilities.getString(options, "separator", Character.toString(this.separator));
final String lineSeparator = options == null ? CSVWriter.DEFAULT_LINE_END : final String lineSeparator = options == null ? CSVWriter.DEFAULT_LINE_END :
JSONUtilities.getString(options, "lineSeparator", CSVWriter.DEFAULT_LINE_END); JSONUtilities.getString(options, "lineSeparator", CSVWriter.DEFAULT_LINE_END);
final boolean quoteAll = options == null ? false : JSONUtilities.getBoolean(options, "quoteAll", false);
final boolean printColumnHeader = final boolean printColumnHeader =
(params != null && params.getProperty("printColumnHeader") != null) ? (params != null && params.getProperty("printColumnHeader") != null) ?
@ -110,7 +111,7 @@ public class CsvExporter implements WriterExporter{
cellData.text : cellData.text :
""; "";
} }
csvWriter.writeNext(strings, false); csvWriter.writeNext(strings, quoteAll);
} }
} }
}; };

2
main/src/com/google/refine/importers/FixedWidthImporter.java
View File

@ -185,7 +185,7 @@ public class FixedWidthImporter extends TabularImportingParserBase {
} }
} }
if (counts != null) { if (counts != null && lineCount > 2) {
List<Integer> widths = new ArrayList<Integer>(); List<Integer> widths = new ArrayList<Integer>();
int startIndex = 0; int startIndex = 0;

4
main/src/com/google/refine/importers/ImporterUtilities.java
View File

@ -57,10 +57,6 @@ public class ImporterUtilities {
static public Serializable parseCellValue(String text) { static public Serializable parseCellValue(String text) {
if (text.length() > 0) { if (text.length() > 0) {
if (text.length() > 1 && text.startsWith("\"") && text.endsWith("\"")) {
return text.substring(1, text.length() - 1);
}
String text2 = text.trim(); String text2 = text.trim();
if (text2.length() > 0) { if (text2.length() > 0) {
try { try {

29
main/src/com/google/refine/importers/JsonImporter.java
View File

@ -34,10 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers; package com.google.refine.importers;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Reader;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
@ -67,7 +65,7 @@ public class JsonImporter extends TreeImportingParserBase {
public final static String ANONYMOUS = "_"; public final static String ANONYMOUS = "_";
public JsonImporter() { public JsonImporter() {
super(false); super(true);
} }
static private class PreviewParsingState { static private class PreviewParsingState {
@ -84,18 +82,13 @@ public class JsonImporter extends TreeImportingParserBase {
try { try {
JSONObject firstFileRecord = fileRecords.get(0); JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord); File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file); JsonFactory factory = new JsonFactory();
try { JsonParser parser = factory.createJsonParser(file);
JsonFactory factory = new JsonFactory();
JsonParser parser = factory.createJsonParser(is);
PreviewParsingState state = new PreviewParsingState(); PreviewParsingState state = new PreviewParsingState();
Object rootValue = parseForPreview(parser, state); Object rootValue = parseForPreview(parser, state);
if (rootValue != null) { if (rootValue != null) {
JSONUtilities.safePut(options, "dom", rootValue); JSONUtilities.safePut(options, "dom", rootValue);
}
} finally {
is.close();
} }
} catch (IOException e) { } catch (IOException e) {
logger.error("Error generating parser UI initialization data for JSON file", e); logger.error("Error generating parser UI initialization data for JSON file", e);
@ -201,11 +194,11 @@ public class JsonImporter extends TreeImportingParserBase {
@Override @Override
public void parseOneFile(Project project, ProjectMetadata metadata, public void parseOneFile(Project project, ProjectMetadata metadata,
ImportingJob job, String fileSource, Reader reader, ImportingJob job, String fileSource, InputStream is,
ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List<Exception> exceptions) { ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List<Exception> exceptions) {
parseOneFile(project, metadata, job, fileSource, parseOneFile(project, metadata, job, fileSource,
new JSONTreeReader(reader), rootColumnGroup, limit, options, exceptions); new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
} }
static public class JSONTreeReader implements TreeReader { static public class JSONTreeReader implements TreeReader {
@ -220,9 +213,9 @@ public class JsonImporter extends TreeImportingParserBase {
private Serializable fieldValue = null; private Serializable fieldValue = null;
public JSONTreeReader(Reader reader) { public JSONTreeReader(InputStream is) {
try { try {
parser = factory.createJsonParser(reader); parser = factory.createJsonParser(is);
current = null; current = null;
next = parser.nextToken(); next = parser.nextToken();
} catch (IOException e) { } catch (IOException e) {

2
main/src/com/google/refine/importers/LineBasedFormatGuesser.java
View File

@ -13,7 +13,7 @@ public class LineBasedFormatGuesser implements FormatGuesser {
return "text/line-based/*sv"; return "text/line-based/*sv";
} }
int[] widths = FixedWidthImporter.guessColumnWidths(file, encoding); int[] widths = FixedWidthImporter.guessColumnWidths(file, encoding);
if (widths != null) { if (widths != null && widths.length > 0) {
return "text/line-based/fixed-width"; return "text/line-based/fixed-width";
} }
return null; return null;

12
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -42,6 +42,7 @@ import java.io.LineNumberReader;
import java.io.Reader; import java.io.Reader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
@ -95,12 +96,13 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
} }
sep = StringEscapeUtils.unescapeJava(sep); sep = StringEscapeUtils.unescapeJava(sep);
boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true); boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true);
boolean strictQuotes = JSONUtilities.getBoolean(options, "strictQuotes", false);
final CSVParser parser = new CSVParser( final CSVParser parser = new CSVParser(
sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators. sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators.
CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_QUOTE_CHARACTER,
(char) 127, // we don't want escape processing try DEL as a rare character until we can turn it off (char) 127, // we don't want escape processing try DEL as a rare character until we can turn it off
CSVParser.DEFAULT_STRICT_QUOTES, strictQuotes,
CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE,
!processQuotes); !processQuotes);
@ -126,14 +128,10 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
ArrayList<Object> cells = new ArrayList<Object>(); ArrayList<Object> cells = new ArrayList<Object>();
String[] tokens = parser.parseLineMulti(line); String[] tokens = parser.parseLineMulti(line);
for (String s : tokens){ cells.addAll(Arrays.asList(tokens));
cells.add(s);
}
while (parser.isPending()) { while (parser.isPending()) {
tokens = parser.parseLineMulti(lnReader.readLine()); tokens = parser.parseLineMulti(lnReader.readLine());
for (String s : tokens) { cells.addAll(Arrays.asList(tokens));
cells.add(s);
}
} }
return cells; return cells;
} }

65
main/src/com/google/refine/importers/tree/ImportRecord.java
View File

@ -1,6 +1,6 @@
package com.google.refine.importers.tree; package com.google.refine.importers.tree;
import java.util.LinkedList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
@ -10,5 +10,66 @@ import com.google.refine.model.Cell;
* *
*/ */
public class ImportRecord { public class ImportRecord {
public List<List<Cell>> rows = new LinkedList<List<Cell>>(); public List<List<Cell>> rows = new BasedList<List<Cell>>();
/**
* A List implementation to match the characteristics needed by the
* import process. It's optimized for a relatively small number of
* contiguous records at a potentially large offset from zero.
* <p>
* I suspect it's usually only a single row, but we support more, just
* not as efficiently. Depending on the behavior of the ColumnGroups
* this may not be necessary at all, but I don't fully understand what it
* does, so we'll just put this hack in place for now.
*
* @param <T>
*/
class BasedList<T> extends ArrayList<T> {
private static final long serialVersionUID = 1L;
int offset = Integer.MAX_VALUE;
public T set(int index, T element) {
rebase(index);
extend(index);
return super.set(index - offset, element);
}
public T get(int index) {
if (offset == Integer.MAX_VALUE || index - offset > size() - 1) {
return null;
}
return super.get(index - offset);
}
private void rebase(final int index) {
if (index < offset) {
if (offset < Integer.MAX_VALUE) {
int new_offset = Math.max(0, index - 10); // Leave some extra room
int delta = offset - new_offset;
// Ensure room at top
for (int i = 0; i < delta; i++) {
add(null);
}
// Shuffle up
for (int i = size(); i > delta; i --) {
set(i,get(i-delta));
} // Null unused entries
for (int i = 0; i < delta; i++) {
set(i,null);
}
offset = new_offset;
} else {
offset = index;
}
}
}
private void extend(final int index) {
int i = index - offset;
while (i >= size()) {
add(null);
}
}
}
} }

8
main/src/com/google/refine/importers/tree/TreeImportUtilities.java
View File

@ -150,11 +150,13 @@ public abstract class TreeImportUtilities {
int cellIndex = column.cellIndex; int cellIndex = column.cellIndex;
int rowIndex = Math.max(columnGroup.nextRowIndex, column.nextRowIndex); int rowIndex = Math.max(columnGroup.nextRowIndex, column.nextRowIndex);
while (rowIndex >= record.rows.size()) {
record.rows.add(new ArrayList<Cell>());
}
List<Cell> row = record.rows.get(rowIndex); List<Cell> row = record.rows.get(rowIndex);
if (row == null) {
row = new ArrayList<Cell>();
record.rows.set(rowIndex, row);
}
while (cellIndex >= row.size()) { while (cellIndex >= row.size()) {
row.add(null); row.add(null);
} }

50
main/src/com/google/refine/importers/tree/XmlImportUtilities.java
View File

@ -129,7 +129,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
* null if no candidates were found (less than 6 recurrences) * null if no candidates were found (less than 6 recurrences)
*/ */
static public String[] detectRecordElement(TreeReader parser) { static public String[] detectRecordElement(TreeReader parser) {
logger.trace("detectRecordElement(inputStream)"); if (logger.isTraceEnabled()) {
logger.trace("detectRecordElement(inputStream)");
}
List<RecordElementCandidate> candidates = new ArrayList<RecordElementCandidate>(); List<RecordElementCandidate> candidates = new ArrayList<RecordElementCandidate>();
try { try {
@ -161,7 +163,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
} }
static protected RecordElementCandidate detectRecordElement(TreeReader parser, String[] path) { static protected RecordElementCandidate detectRecordElement(TreeReader parser, String[] path) {
logger.trace("detectRecordElement(TreeReader, String[])"); if (logger.isTraceEnabled()) {
logger.trace("detectRecordElement(TreeReader, String[])");
}
List<RecordElementCandidate> descendantCandidates = new ArrayList<RecordElementCandidate>(); List<RecordElementCandidate> descendantCandidates = new ArrayList<RecordElementCandidate>();
Map<String, Integer> immediateChildCandidateMap = new HashMap<String, Integer>(); Map<String, Integer> immediateChildCandidateMap = new HashMap<String, Integer>();
@ -217,7 +221,10 @@ public class XmlImportUtilities extends TreeImportUtilities {
sortRecordElementCandidates(immediateChildCandidates); sortRecordElementCandidates(immediateChildCandidates);
RecordElementCandidate ourCandidate = immediateChildCandidates.get(0); RecordElementCandidate ourCandidate = immediateChildCandidates.get(0);
logger.trace("ourCandidate.count : " + ourCandidate.count + "; immediateChildCandidates.size() : " + immediateChildCandidates.size()); if (logger.isTraceEnabled()) {
logger.trace("ourCandidate.count : " + ourCandidate.count + "; immediateChildCandidates.size() : "
+ immediateChildCandidates.size());
}
if (ourCandidate.count / immediateChildCandidates.size() > 5) { if (ourCandidate.count / immediateChildCandidates.size() > 5) {
return ourCandidate; return ourCandidate;
} }
@ -259,7 +266,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean storeEmptyStrings, boolean storeEmptyStrings,
boolean guessDataType boolean guessDataType
) { ) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)"); if (logger.isTraceEnabled()) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)");
}
try { try {
while (parser.hasNext()) { while (parser.hasNext()) {
Token eventType = parser.next(); Token eventType = parser.next();
@ -305,8 +314,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean storeEmptyStrings, boolean storeEmptyStrings,
boolean guessDataType boolean guessDataType
) throws TreeReaderException { ) throws TreeReaderException {
logger.trace("findRecord(Project, TreeReader, String[], int, ImportColumnGroup - path:"+Arrays.toString(recordPath)); if (logger.isTraceEnabled()) {
logger.trace("findRecord(Project, TreeReader, String[], int, ImportColumnGroup - path:"+Arrays.toString(recordPath));
}
if(parser.current() == Token.Ignorable){//XMLStreamConstants.START_DOCUMENT){ if(parser.current() == Token.Ignorable){//XMLStreamConstants.START_DOCUMENT){
logger.warn("Cannot use findRecord method for START_DOCUMENT event"); logger.warn("Cannot use findRecord method for START_DOCUMENT event");
return; return;
@ -382,7 +392,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean storeEmptyStrings, boolean storeEmptyStrings,
boolean guessDataType boolean guessDataType
) throws TreeReaderException { ) throws TreeReaderException {
logger.trace("processRecord(Project,TreeReader,ImportColumnGroup)"); if (logger.isTraceEnabled()) {
logger.trace("processRecord(Project,TreeReader,ImportColumnGroup)");
}
ImportRecord record = new ImportRecord(); ImportRecord record = new ImportRecord();
processSubRecord(project, parser, rootColumnGroup, record, 0, trimStrings, storeEmptyStrings, guessDataType); processSubRecord(project, parser, rootColumnGroup, record, 0, trimStrings, storeEmptyStrings, guessDataType);
@ -418,8 +430,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean storeEmptyStrings, boolean storeEmptyStrings,
boolean guessDataType boolean guessDataType
) throws TreeReaderException { ) throws TreeReaderException {
logger.trace("processFieldAsRecord(Project,TreeReader,ImportColumnGroup)"); if (logger.isTraceEnabled()) {
logger.trace("processFieldAsRecord(Project,TreeReader,ImportColumnGroup)");
}
Serializable value = parser.getValue(); Serializable value = parser.getValue();
ImportRecord record = null; ImportRecord record = null;
if (value instanceof String) { if (value instanceof String) {
@ -455,20 +468,19 @@ public class XmlImportUtilities extends TreeImportUtilities {
} }
static protected void addImportRecordToProject(ImportRecord record, Project project) { static protected void addImportRecordToProject(ImportRecord record, Project project) {
if (record.rows.size() > 0) { for (List<Cell> row : record.rows) {
for (List<Cell> row : record.rows) { if (row.size() > 0) {
Row realRow = new Row(row.size()); Row realRow = null;
int cellCount = 0;
for (int c = 0; c < row.size(); c++) { for (int c = 0; c < row.size(); c++) {
Cell cell = row.get(c); Cell cell = row.get(c);
if (cell != null) { if (cell != null) {
if (realRow == null) {
realRow = new Row(row.size());
}
realRow.setCell(c, cell); realRow.setCell(c, cell);
cellCount++;
} }
} }
if (realRow != null) {
if (cellCount > 0) {
project.rows.add(realRow); project.rows.add(realRow);
} }
} }
@ -506,7 +518,9 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean storeEmptyStrings, boolean storeEmptyStrings,
boolean guessDataType boolean guessDataType
) throws TreeReaderException { ) throws TreeReaderException {
logger.trace("processSubRecord(Project,TreeReader,ImportColumnGroup,ImportRecord) lvl:"+level+" "+columnGroup); if (logger.isTraceEnabled()) {
logger.trace("processSubRecord(Project,TreeReader,ImportColumnGroup,ImportRecord) lvl:"+level+" "+columnGroup);
}
if(parser.current() == Token.Ignorable) { if(parser.current() == Token.Ignorable) {
return; return;

3
main/src/com/google/refine/importing/ImportingManager.java
View File

@ -270,6 +270,9 @@ public class ImportingManager {
static public String getFormat(String fileName, String mimeType) { static public String getFormat(String fileName, String mimeType) {
String fileNameFormat = getFormatFromFileName(fileName); String fileNameFormat = getFormatFromFileName(fileName);
if (mimeType != null) {
mimeType = mimeType.split(";")[0];
}
String mimeTypeFormat = mimeType == null ? null : getFormatFromMimeType(mimeType); String mimeTypeFormat = mimeType == null ? null : getFormatFromMimeType(mimeType);
if (mimeTypeFormat == null) { if (mimeTypeFormat == null) {
return fileNameFormat; return fileNameFormat;

2
main/src/com/google/refine/model/Recon.java
View File

@ -253,7 +253,7 @@ public class Recon implements HasFields, Jsonizable {
@Override @Override
public Object getField(String name, Properties bindings) { public Object getField(String name, Properties bindings) {
int index = s_featureMap.containsKey(name) ? s_featureMap.get(name) : -1; int index = s_featureMap.containsKey(name) ? s_featureMap.get(name) : -1;
return (index > 0 && index < features.length) ? features[index] : null; return (index >= 0 && index < features.length) ? features[index] : null;
} }
@Override @Override

10
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -44,6 +44,8 @@ import java.util.Properties;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable; import com.google.refine.Jsonizable;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
@ -54,6 +56,8 @@ import com.google.refine.model.Row;
import edu.mit.simile.butterfly.ButterflyModule; import edu.mit.simile.butterfly.ButterflyModule;
abstract public class ReconConfig implements Jsonizable { abstract public class ReconConfig implements Jsonizable {
final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config");
static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass = static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass =
new HashMap<String, List<Class<? extends ReconConfig>>>(); new HashMap<String, List<Class<? extends ReconConfig>>>();
@ -96,7 +100,7 @@ abstract public class ReconConfig implements Jsonizable {
} }
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); LOGGER.error("Reconstruct failed",e);
} }
return null; return null;
} }
@ -122,7 +126,7 @@ abstract public class ReconConfig implements Jsonizable {
try { try {
write(jsonWriter, new Properties()); write(jsonWriter, new Properties());
} catch (JSONException e) { } catch (JSONException e) {
e.printStackTrace(); LOGGER.error("Save failed",e);
} }
} }
} }

1
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -456,6 +456,7 @@ public class StandardReconConfig extends ReconConfig {
static final protected Set<String> s_stopWords = new HashSet<String>(); static final protected Set<String> s_stopWords = new HashSet<String>();
static { static {
// FIXME: This is English specific
s_stopWords.add("the"); s_stopWords.add("the");
s_stopWords.add("a"); s_stopWords.add("a");
s_stopWords.add("and"); s_stopWords.add("and");

9
main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.LinkedList; import java.util.LinkedList;
@ -228,7 +227,7 @@ public class JsonImporterTests extends ImporterTest {
String sampleJson2 = "{\"field\":{}}"; String sampleJson2 = "{\"field\":{}}";
String sampleJson3 = "{\"field\":[{},{}]}"; String sampleJson3 = "{\"field\":[{},{}]}";
JSONTreeReader parser = new JSONTreeReader(new StringReader(sampleJson)); JSONTreeReader parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson.getBytes("UTF-8")));
Token token = Token.Ignorable; Token token = Token.Ignorable;
int i = 0; int i = 0;
try{ try{
@ -248,7 +247,7 @@ public class JsonImporterTests extends ImporterTest {
} }
parser = new JSONTreeReader(new StringReader(sampleJson2)); parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson2.getBytes("UTF-8")));
token = Token.Ignorable; token = Token.Ignorable;
i = 0; i = 0;
try{ try{
@ -267,7 +266,7 @@ public class JsonImporterTests extends ImporterTest {
//silent //silent
} }
parser = new JSONTreeReader(new StringReader(sampleJson3)); parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson3.getBytes("UTF-8")));
token = Token.Ignorable; token = Token.Ignorable;
i = 0; i = 0;
try{ try{
@ -505,7 +504,7 @@ public class JsonImporterTests extends ImporterTest {
} }
try { try {
parseOneFile(SUT, inputStream, options); parseOneInputStream(SUT, inputStream, options);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }

5
main/tests/server/src/com/google/refine/tests/importers/XmlImportUtilitiesTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -197,7 +196,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
loadSampleJson(); loadSampleJson();
String[] path = XmlImportUtilitiesStub.detectRecordElement( String[] path = XmlImportUtilitiesStub.detectRecordElement(
new JSONTreeReader(new InputStreamReader(inputStream))); new JSONTreeReader(inputStream));
Assert.assertNotNull(path); Assert.assertNotNull(path);
Assert.assertEquals(path.length, 2); Assert.assertEquals(path.length, 2);
Assert.assertEquals(path[0], JsonImporter.ANONYMOUS); Assert.assertEquals(path[0], JsonImporter.ANONYMOUS);
@ -455,7 +454,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
} }
} }
public TreeReader createJsonParser(){ public TreeReader createJsonParser(){
parser = new JSONTreeReader(new InputStreamReader(inputStream)); parser = new JSONTreeReader(inputStream);
return parser; return parser;
} }
} }

BIN
main/webapp/WEB-INF/lib/jackson-core-asl-1.9.9.jar → main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar
View File

Binary file not shown.

2
main/webapp/modules/core/externals/suggest/suggest-4_1.js vendored
View File

@ -87,7 +87,7 @@
var SEARCH_PARAMS = { var SEARCH_PARAMS = {
key:1, filter:1, spell:1, exact:1, key:1, filter:1, spell:1, exact:1,
lang:1, scoring:1, prefixed:1, stemmed:1, format:1, mql_output:1, lang:1, scoring:1, prefixed:1, stemmed:1, format:1, mql_output:1,
output:1 output:1, type:1
}; };
$.suggest = function(name, prototype) { $.suggest = function(name, prototype) {

2
main/webapp/modules/core/externals/suggest/suggest-4_1.min.js vendored
View File

@ -34,7 +34,7 @@
* *
*/ */
(function(c,q){if(!("console"in window)){var o=window.console={};o.log=o.warn=o.error=o.debug=function(){}}c(function(){var a=c("<div>");c(document.body).append(a);var b=setTimeout(function(){if(c.cleanData){var a=c.cleanData;c.cleanData=function(b){for(var e=0,g;null!=(g=b[e]);e++)c(g).triggerHandler("remove");a(b)}}else{var b=c.fn.remove;c.fn.remove=function(a,f){return this.each(function(){f||(!a||c.filter(a,[this]).length)&&c("*",this).add([this]).each(function(){c(this).triggerHandler("remove")}); (function(c,q){if(!("console"in window)){var o=window.console={};o.log=o.warn=o.error=o.debug=function(){}}c(function(){var a=c("<div>");c(document.body).append(a);var b=setTimeout(function(){if(c.cleanData){var a=c.cleanData;c.cleanData=function(b){for(var e=0,g;null!=(g=b[e]);e++)c(g).triggerHandler("remove");a(b)}}else{var b=c.fn.remove;c.fn.remove=function(a,f){return this.each(function(){f||(!a||c.filter(a,[this]).length)&&c("*",this).add([this]).each(function(){c(this).triggerHandler("remove")});
return b.call(c(this),a,f)})}}},1);a.bind("remove",function(){clearTimeout(b)});a.remove()});var p={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1};c.suggest=function(a,b){c.fn[a]=function(b){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+ return b.call(c(this),a,f)})}}},1);a.bind("remove",function(){clearTimeout(b)});a.remove()});var p={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1,type:1};c.suggest=function(a,b){c.fn[a]=function(b){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+
this.nodeName));var g=c.data(this,a);g&&g._destroy();c.data(this,a,new c.suggest[a](this,b))._init()})};c.suggest[a]=function(b,g){var d=this,e=this.options=c.extend(!0,{},c.suggest.defaults,c.suggest[a].defaults,g),j=e.css_prefix=e.css_prefix||"",h=e.css;this.name=a;c.each(h,function(a){h[a]=j+h[a]});e.ac_param={};c.each(p,function(a){var b=e[a];null===b||""===b||(e.ac_param[a]=b)});e.flyout_lang=null;if(e.ac_param.lang){var i=e.ac_param.lang;"string"===c.type(i)&&(i=i.split(","));if(c.isArray(i)&& this.nodeName));var g=c.data(this,a);g&&g._destroy();c.data(this,a,new c.suggest[a](this,b))._init()})};c.suggest[a]=function(b,g){var d=this,e=this.options=c.extend(!0,{},c.suggest.defaults,c.suggest[a].defaults,g),j=e.css_prefix=e.css_prefix||"",h=e.css;this.name=a;c.each(h,function(a){h[a]=j+h[a]});e.ac_param={};c.each(p,function(a){var b=e[a];null===b||""===b||(e.ac_param[a]=b)});e.flyout_lang=null;if(e.ac_param.lang){var i=e.ac_param.lang;"string"===c.type(i)&&(i=i.split(","));if(c.isArray(i)&&
i.length&&(i=c.trim(i[0])))e.flyout_lang=i}this._status={START:"",LOADING:"",SELECT:"",ERROR:""};e.status&&(e.status instanceof Array&&3<=e.status.length)&&(this._status.START=e.status[0]||"",this._status.LOADING=e.status[1]||"",this._status.SELECT=e.status[2]||"",4===e.status.length&&(this._status.ERROR=e.status[3]||""));var i=this.status=c('<div style="display:none;">').addClass(h.status),k=this.list=c("<ul>").addClass(h.list),l=this.pane=c('<div style="display:none;" class="fbs-reset">').addClass(h.pane); i.length&&(i=c.trim(i[0])))e.flyout_lang=i}this._status={START:"",LOADING:"",SELECT:"",ERROR:""};e.status&&(e.status instanceof Array&&3<=e.status.length)&&(this._status.START=e.status[0]||"",this._status.LOADING=e.status[1]||"",this._status.SELECT=e.status[2]||"",4===e.status.length&&(this._status.ERROR=e.status[3]||""));var i=this.status=c('<div style="display:none;">').addClass(h.status),k=this.list=c("<ul>").addClass(h.list),l=this.pane=c('<div style="display:none;" class="fbs-reset">').addClass(h.pane);
l.append(i).append(k);e.parent?c(e.parent).append(l):(l.css("position","absolute"),e.zIndex&&l.css("z-index",e.zIndex),c(document.body).append(l));l.bind("mousedown",function(a){d.input.data("dont_hide",true);a.stopPropagation()}).bind("mouseup",function(a){d.input.data("dont_hide")&&d.input.focus();d.input.removeData("dont_hide");a.stopPropagation()}).bind("click",function(a){a.stopPropagation();if(a=d.get_selected()){d.onselect(a,true);d.hide_all()}});k.hover(function(a){d.hoverover_list(a)},function(a){d.hoverout_list(a)}); l.append(i).append(k);e.parent?c(e.parent).append(l):(l.css("position","absolute"),e.zIndex&&l.css("z-index",e.zIndex),c(document.body).append(l));l.bind("mousedown",function(a){d.input.data("dont_hide",true);a.stopPropagation()}).bind("mouseup",function(a){d.input.data("dont_hide")&&d.input.focus();d.input.removeData("dont_hide");a.stopPropagation()}).bind("click",function(a){a.stopPropagation();if(a=d.get_selected()){d.onselect(a,true);d.hide_all()}});k.hover(function(a){d.hoverover_list(a)},function(a){d.hoverout_list(a)});

2
main/webapp/modules/core/scripts/dialogs/clustering-dialog.html
View File

@ -9,7 +9,7 @@
"New York" and "new york" are very likely to refer to the same concept and "New York" and "new york" are very likely to refer to the same concept and
just have capitalization differences, and "Gödel" and "Godel" probably refer just have capitalization differences, and "Gödel" and "Godel" probably refer
to the same person. to the same person.
<a href="http://code.google.com/p/google-refine/wiki/Clustering" target="_blank">Find out more ...</a> <a href="https://github.com/OpenRefine/OpenRefine/wiki/Clustering" target="_blank">Find out more ...</a>
</td> </td>
</tr> </tr>
<tr> <tr>

4
main/webapp/modules/core/scripts/dialogs/clustering-dialog.js
View File

@ -153,7 +153,7 @@ ClusteringDialog.prototype._renderTable = function(clusters) {
cluster.value = value; cluster.value = value;
parent.find("input[type='text']").val(value); parent.find("input[type='text']").val(value);
parent.find("input:not(:checked)").attr('checked', true).change(); parent.find("input[type='checkbox']").attr('checked', true).change();
return false; return false;
}; };
for (var c = 0; c < choices.length; c++) { for (var c = 0; c < choices.length; c++) {
@ -202,7 +202,7 @@ ClusteringDialog.prototype._renderTable = function(clusters) {
editCheck.attr("checked", "true"); editCheck.attr("checked", "true");
} }
var input = $('<input size="25" />') var input = $('<input type="text" size="25" />')
.attr("value", cluster.value) .attr("value", cluster.value)
.keyup(function() { .keyup(function() {
cluster.value = this.value; cluster.value = this.value;

2
main/webapp/modules/core/scripts/project.js
View File

@ -35,7 +35,7 @@ var theProject;
var ui = {}; var ui = {};
var Refine = { var Refine = {
refineHelperService: "http://1.refine-helper.stefanomazzocchi.user.dev.freebaseapps.com" refineHelperService: "http://openrefine-helper.freebaseapps.com"
}; };
Refine.reportException = function(e) { Refine.reportException = function(e) {

4
main/webapp/modules/core/scripts/reconciliation/recon-dialog.js
View File

@ -214,8 +214,8 @@ ReconDialog.prototype._onAddNamespacedService = function() {
var namespaceData = elmts.namespaceInput.data("data.suggest"); var namespaceData = elmts.namespaceInput.data("data.suggest");
var typeData = elmts.typeInput.data("data.suggest"); var typeData = elmts.typeInput.data("data.suggest");
if (namespaceData) { if (namespaceData) {
var url = "http://4.standard-reconcile.dfhuynh.user.dev.freebaseapps.com/namespace_reconcile?namespace=" + var url = "http://reconcile.freebaseapps.com/namespace_reconcile?namespace="
escape(namespaceData.id); + escape(namespaceData.id);
if (typeData) { if (typeData) {
url += "&type=" + typeData.id; url += "&type=" + typeData.id;
} }

4
main/webapp/modules/core/scripts/reconciliation/recon-manager.js
View File

@ -156,7 +156,9 @@ ReconciliationManager.save = function(f) {
ReconciliationManager._rebuildMap(); ReconciliationManager._rebuildMap();
} else { } else {
ReconciliationManager.registerStandardService( ReconciliationManager.registerStandardService(
"http://4.standard-reconcile.dfhuynh.user.dev.freebaseapps.com/reconcile"); // "http://reconcile.freebaseapps.com/reconcile"
"http://standard-reconcile.freebaseapps.com/reconcile"
);
} }
}, },
dataType: "json" dataType: "json"

2
main/webapp/modules/core/scripts/reconciliation/standard-service-panel.js
View File

@ -245,7 +245,7 @@ ReconStandardServicePanel.prototype._rewirePropertySuggests = function(type) {
} else if (this._isInFreebaseSchemaSpace()) { } else if (this._isInFreebaseSchemaSpace()) {
var namespace = (type) ? (typeof type == "string" ? type : type.id) : "/common/topic" var namespace = (type) ? (typeof type == "string" ? type : type.id) : "/common/topic"
inputs.suggestP({ inputs.suggestP({
filter : '(all type:/type/property (any namespace:/type/object namespace:' + namespace + '))' filter : '(should (any namespace:/type/object namespace:' + namespace + '))'
}); });
} }
}; };

8
main/webapp/modules/core/scripts/util/custom-suggest.js
View File

@ -33,6 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
var CustomSuggest = {}; var CustomSuggest = {};
// Default API key for Refine to use for freebase suggest widget
//CustomSuggest.setFreebaseAPIKey("AIzaSyBBTAtJ31v_jlg_ImbQuBNnAaAyrHzRyW8"); // Google key
CustomSuggest.FREEBASE_API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M"; // OpenRefine default key
CustomSuggest.setFreebaseAPIKey = function(freebaseAPIKey) { CustomSuggest.setFreebaseAPIKey = function(freebaseAPIKey) {
$.suggest.suggest.defaults.key = freebaseAPIKey; $.suggest.suggest.defaults.key = freebaseAPIKey;
$.suggest.suggestT.defaults.key = freebaseAPIKey; $.suggest.suggestT.defaults.key = freebaseAPIKey;
@ -111,6 +115,7 @@ CustomSuggest.setFreebaseAPIKey = function(freebaseAPIKey) {
{}, {},
$.suggest.suggest.defaults, { $.suggest.suggest.defaults, {
scoring: "schema", scoring: "schema",
type: "/type/property",
css: { pane: "fbs-pane fbs-pane-property" } css: { pane: "fbs-pane fbs-pane-property" }
} }
) )
@ -172,8 +177,7 @@ CustomSuggest.setFreebaseAPIKey = function(freebaseAPIKey) {
if (data.value && data.value != "null") { if (data.value && data.value != "null") {
CustomSuggest.setFreebaseAPIKey(data.value); CustomSuggest.setFreebaseAPIKey(data.value);
} else { } else {
// Default API key for Refine to use for freebase suggest widget CustomSuggest.setFreebaseAPIKey(CustomSuggest.FREEBASE_API_KEY);
CustomSuggest.setFreebaseAPIKey("AIzaSyBBTAtJ31v_jlg_ImbQuBNnAaAyrHzRyW8");
} }
} }
} }

2
main/webapp/modules/core/scripts/views/data-table/cell-ui.js
View File

@ -379,7 +379,7 @@ DataTableCellUI.prototype._searchForMatch = function(suggestOptions) {
var suggestOptions2 = $.extend({ align: "left" }, suggestOptions var suggestOptions2 = $.extend({ align: "left" }, suggestOptions
|| { all_types: true, // FIXME: all_types isn't documented for Suggest. Is it still implemented? || { all_types: true, // FIXME: all_types isn't documented for Suggest. Is it still implemented?
filter: "(not (any type:/common/document type:/type/content))" }); // blacklist documents and images filter: "(not (any type:/common/document type:/type/content type:/common/resource))" }); // blacklist documents and images
if (suggestOptions2.service_url) { if (suggestOptions2.service_url) {
// Old style suggest API // Old style suggest API
suggestOptions2.key = null; suggestOptions2.key = null;

4
main/webapp/modules/core/scripts/views/data-table/menu-reconcile.js
View File

@ -89,8 +89,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
"id" : data.id, "id" : data.id,
"type" : [] "type" : []
}; };
var baseUrl = "http://api.freebase.com/api/service/mqlread"; var baseUrl = "https://www.googleapis.com/freebase/v1/mqlread?key=" + Freebase.API_KEY + "&";
var url = baseUrl + "?" + $.param({ query: JSON.stringify({ query: query }) }) + "&callback=?"; var url = baseUrl + $.param({ query: JSON.stringify(query) }) + "&callback=?";
$.getJSON( $.getJSON(
url, url,

111
refine
View File

@ -46,6 +46,8 @@ where [options] include:
-m <memory> max memory heap size to use -m <memory> max memory heap size to use
default: 1024M default: 1024M
-k <google api key> a server API key for calling Google APIs
-v <level> verbosity level [from low to high: error,warn,info,debug,trace] -v <level> verbosity level [from low to high: error,warn,info,debug,trace]
default: info default: info
@ -151,8 +153,8 @@ check_running() {
CHECK_STR="<title>OpenRefine</title>" CHECK_STR="<title>OpenRefine</title>"
if [ "$CURL" ] ; then if [ "$CURL" ] ; then
curl -s -S $URL > /dev/null 2>&1 curl -s -S -f $URL > /dev/null 2>&1
if [ "$?" = "7" ] ; then if [ "$?" = "7" ] || [ "$?" = "22" ] ; then
NOT_RUNNING="1" NOT_RUNNING="1"
fi fi
elif [ "$WGET" ] ; then elif [ "$WGET" ] ; then
@ -207,12 +209,11 @@ get_revision() {
if [ -d ".svn" ] ; then if [ -d ".svn" ] ; then
INFO=`svn info` INFO=`svn info`
elif [ -d ".git" ] ; then elif [ -d ".git" ] ; then
INFO=`git svn info` INFO=`git describe`
else else
error "cannot obtain revision, exiting!" error "cannot obtain revision, exiting!"
fi fi
REVISION=`echo $INFO | sed s/^$VERSION-//`
REVISION=`echo $INFO | sed 's/.*Revision: /r/' | sed 's/ .*//'`
} }
download() { download() {
@ -263,6 +264,17 @@ load_data() {
fi fi
} }
display() {
FILE=$1
if [ "$OS" = "macosx" ] ; then
open $FILE
elif [ "$OS" = "linux" ] ; then
gnome-open $FILE
else
notepad $FILE
fi
}
# ---------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------
build_prepare() { build_prepare() {
@ -286,9 +298,10 @@ tools_prepare() {
ant_prepare() { ant_prepare() {
tools_prepare tools_prepare
ANT_URL="http://archive.apache.org/dist/ant/binaries/apache-ant-1.8.1-bin.tar.gz" ANT_VERSION="1.9.0"
ANT_URL="http://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz"
ANT_FILE=`echo $ANT_URL | sed 's|.*/||'` ANT_FILE=`echo $ANT_URL | sed 's|.*/||'`
ANT_DIR="apache-ant-1.8.1" ANT_DIR="apache-ant-${ANT_VERSION}"
ANT="`which ant 2> /dev/null`" ANT="`which ant 2> /dev/null`"
if [ -z "$ANT" ] ; then if [ -z "$ANT" ] ; then
@ -325,12 +338,14 @@ appengine_prepare() {
launch4j_prepare() { launch4j_prepare() {
tools_prepare tools_prepare
L4J_VERSION="3.1.0-beta1" # buggy!
L4J_VERSION="3.0.2"
if [ "$OS" = "macosx" ] ; then if [ "$OS" = "macosx" ] ; then
LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/3.0.2/launch4j-3.0.2-macosx.tgz" LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/${L4J_VERSION}/launch4j-${L4J_VERSION}-macosx.tgz"
elif [ "$OS" = "windows" ] ; then elif [ "$OS" = "windows" ] ; then
LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/3.0.2/launch4j-3.0.2-win32.zip" LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/${L4J_VERSION}/launch4j-${L4J_VERSION}-win32.zip"
elif [ "$OS" = "linux" ] ; then elif [ "$OS" = "linux" ] ; then
LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/3.0.2/launch4j-3.0.2-linux.tgz" LAUNCH4J_URL="http://downloads.sourceforge.net/project/launch4j/launch4j-3/${L4J_VERSION}/launch4j-${L4J_VERSION}-linux.tgz"
fi fi
LAUNCH4J_FILE=`echo $LAUNCH4J_URL | sed 's|.*/||'` LAUNCH4J_FILE=`echo $LAUNCH4J_URL | sed 's|.*/||'`
@ -342,6 +357,7 @@ launch4j_prepare() {
# without PPC support (e.g. Lion) so we patch it to include Intel binaries; # without PPC support (e.g. Lion) so we patch it to include Intel binaries;
# the following lines can be removed once we update to a version of # the following lines can be removed once we update to a version of
# launch4j that fixes the issue (not available yet). # launch4j that fixes the issue (not available yet).
# (3.1.0-beta1 fixes this, but introduces other bugs which make it unusable)
if [ "$OS" = "macosx" ] ; then if [ "$OS" = "macosx" ] ; then
if [ ! -f "$REFINE_TOOLS_DIR/launch4j/bin/windres.bak" ] ; then if [ ! -f "$REFINE_TOOLS_DIR/launch4j/bin/windres.bak" ] ; then
mv "$REFINE_TOOLS_DIR/launch4j/bin/windres" "$REFINE_TOOLS_DIR/launch4j/bin/windres.bak" mv "$REFINE_TOOLS_DIR/launch4j/bin/windres" "$REFINE_TOOLS_DIR/launch4j/bin/windres.bak"
@ -356,20 +372,20 @@ launch4j_prepare() {
fi fi
} }
jarbundler_prepare() { appbundler_prepare() {
tools_prepare tools_prepare
JARBUNDLER_URL="http://www.informagen.com/JarBundler/dist/jarbundler.tar.gz" APPBUNDLER_URL="http://java.net/projects/appbundler/downloads/download/appbundler-1.0.jar"
JARBUNDLER_FILE=`echo $JARBUNDLER_URL | sed 's|.*/||'` APPBUNDLER_DIR="."
JARBUNDLER_DIR="jarbundler-2.2.0" APPBUNDLER_FILE=`echo $APPBUNDLER_URL | sed 's|.*/||'`
tool_download $JARBUNDLER_URL $JARBUNDLER_FILE $JARBUNDLER_DIR tool_download $APPBUNDLER_URL $APPBUNDLER_FILE $APPBUNDLER_DIR
} }
virtualenv_prepare() { virtualenv_prepare() {
check_python check_python
VIRTUALENV_DIR="virtualenv-1.7.1.2" VIRTUALENV_DIR="virtualenv-1.9.1"
VIRTUALENV_FILE="${VIRTUALENV_DIR}.tar.gz" VIRTUALENV_FILE="${VIRTUALENV_DIR}.tar.gz"
VIRTUALENV_URL="http://pypi.python.org/packages/source/v/virtualenv/${VIRTUALENV_FILE}" VIRTUALENV_URL="http://pypi.python.org/packages/source/v/virtualenv/${VIRTUALENV_FILE}"
@ -417,10 +433,10 @@ windmill_prepare() {
findbugs_prepare() { findbugs_prepare() {
tools_prepare tools_prepare
FB_VERSION="2.0.2"
FINDBUGS_URL="http://downloads.sourceforge.net/project/findbugs/findbugs/1.3.9/findbugs-1.3.9.tar.gz" FINDBUGS_URL="http://downloads.sourceforge.net/project/findbugs/findbugs/${FB_VERSION}/findbugs-${FB_VERSION}.tar.gz"
FINDBUGS_FILE=`echo $FINDBUGS_URL | sed 's|.*/||'` FINDBUGS_FILE=`echo $FINDBUGS_URL | sed 's|.*/||'`
FINDBUGS_DIR="findbugs-1.3.9" FINDBUGS_DIR="findbugs-${FB_VERSION}"
tool_download $FINDBUGS_URL $FINDBUGS_FILE $FINDBUGS_DIR tool_download $FINDBUGS_URL $FINDBUGS_FILE $FINDBUGS_DIR
} }
@ -428,9 +444,10 @@ findbugs_prepare() {
pmd_prepare() { pmd_prepare() {
tools_prepare tools_prepare
PMD_URL="http://downloads.sourceforge.net/project/pmd/pmd/4.2.5/pmd-bin-4.2.5.zip" PMD_VERSION="4.2.6"
PMD_FILE="pmd-bin-4.2.5.zip" PMD_URL="http://downloads.sourceforge.net/project/pmd/pmd/${PMD_VERSION}/pmd-bin-${PMD_VERSION}.zip"
PMD_DIR="pmd-4.2.5" PMD_FILE=`echo $PMD_URL | sed 's|.*/||'`
PMD_DIR="pmd-bin-${PMD_VERSION}"
tool_download $PMD_URL $PMD_FILE $PMD_DIR tool_download $PMD_URL $PMD_FILE $PMD_DIR
} }
@ -438,9 +455,10 @@ pmd_prepare() {
jslint_prepare() { jslint_prepare() {
tools_prepare tools_prepare
JSLINT_URL="http://jslint4java.googlecode.com/files/jslint4java-1.3.3-dist.zip" JSLINT_VERSION="2.0.3"
JSLINT_FILE="jslint4java-1.3.3-dist.zip" JSLINT_URL="http://jslint4java.googlecode.com/files/jslint4java-${JSLINT_VERSION}-dist.zip"
JSLINT_DIR="jslint4java-1.3.3" JSLINT_FILE=`echo $JSLINT_URL | sed 's|.*/||'`
JSLINT_DIR="jslint4java-${JSLINT_VERSION}"
tool_download $JSLINT_URL $JSLINT_FILE $JSLINT_DIR tool_download $JSLINT_URL $JSLINT_FILE $JSLINT_DIR
} }
@ -503,15 +521,15 @@ mac_dist() {
get_version $1 get_version $1
get_revision get_revision
jarbundler_prepare appbundler_prepare
ANT_PARAMS="-Djarbundler.dir=${REFINE_TOOLS_DIR}/${JARBUNDLER_DIR}" ANT_PARAMS="-Dappbundler.dir=${REFINE_TOOLS_DIR}/${APPBUNDLER_DIR}"
ant mac ant mac
mkdir -p "$REFINE_BUILD_DIR/mac/.background" mkdir -p "$REFINE_BUILD_DIR/mac/.background"
cp graphics/dmg_background/dmg_background.png "$REFINE_BUILD_DIR/mac/.background/dmg_background.png" cp graphics/dmg_background/dmg_background.png "$REFINE_BUILD_DIR/mac/.background/dmg_background.png"
SIZE=60 SIZE=200
if [ -f "$REFINE_BUILD_DIR/temp_refine.dmg" ] ; then if [ -f "$REFINE_BUILD_DIR/temp_refine.dmg" ] ; then
rm "$REFINE_BUILD_DIR/temp_refine.dmg" rm "$REFINE_BUILD_DIR/temp_refine.dmg"
@ -541,7 +559,7 @@ mac_dist() {
close close
open open
update without registering applications update without registering applications
delay 2 delay 5
eject eject
end tell end tell
end tell end tell
@ -551,14 +569,14 @@ mac_dist() {
sync sync
hdiutil detach $DEVICE hdiutil detach $DEVICE
if [ -f "$REFINE_DIST_DIR/google-refine-$VERSION-$REVISION.dmg" ] ; then if [ -f "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" ] ; then
rm "$REFINE_DIST_DIR/google-refine-$VERSION-$REVISION.dmg" rm "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg"
fi fi
hdiutil convert "$REFINE_BUILD_DIR/temp_refine.dmg" -format UDZO -imagekey zlib-level=9 -o "$REFINE_DIST_DIR/google-refine-$VERSION-$REVISION.dmg" || error "Error compressing DMG" hdiutil convert "$REFINE_BUILD_DIR/temp_refine.dmg" -format UDZO -imagekey zlib-level=9 -o "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" || error "Error compressing DMG"
hdiutil internet-enable -yes "$REFINE_DIST_DIR/google-refine-$VERSION-$REVISION.dmg" || error "Error internet-enabling DMG" hdiutil internet-enable -yes "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" || error "Error internet-enabling DMG"
rm -f "$REFINE_BUILD_DIR/temp_refine.dmg" #rm -f "$REFINE_BUILD_DIR/temp_refine.dmg"
} }
test() { test() {
@ -571,7 +589,7 @@ ui_test() {
windmill_prepare windmill_prepare
REFINE_DATA_DIR="${TMPDIR:=/tmp}/google-refine-tests" REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests"
add_option "-Drefine.headless=true" add_option "-Drefine.headless=true"
@ -634,7 +652,7 @@ run() {
fi fi
if [ ! -d $REFINE_CLASSES_DIR ] ; then if [ ! -d $REFINE_CLASSES_DIR ] ; then
IS_JAR=`ls $REFINE_LIB_DIR | grep google-refine` IS_JAR=`ls $REFINE_LIB_DIR | grep openrefine`
if [ -z "$IS_JAR" ] ; then if [ -z "$IS_JAR" ] ; then
ant build ant build
echo "" echo ""
@ -646,7 +664,7 @@ run() {
fi fi
if [ "$OS" = "macosx" ] ; then if [ "$OS" = "macosx" ] ; then
add_option '-Xdock:icon=graphics/icon/google-refine.icns' add_option '-Xdock:icon=graphics/icon/openrefine.icns'
fi fi
if [ "$REFINE_DATA_DIR" ] ; then if [ "$REFINE_DATA_DIR" ] ; then
@ -665,6 +683,10 @@ run() {
add_option "-Drefine.host=$REFINE_HOST" add_option "-Drefine.host=$REFINE_HOST"
fi fi
if [ "$REFINE_GOOGLE_API_KEY" ] ; then
add_option "-Drefine.google_api_key=$REFINE_GOOGLE_API_KEY"
fi
CLASSPATH="$REFINE_CLASSES_DIR${SEP}$REFINE_LIB_DIR/*" CLASSPATH="$REFINE_CLASSES_DIR${SEP}$REFINE_LIB_DIR/*"
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.refine.Refine" RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.refine.Refine"
@ -760,7 +782,7 @@ findbugs() {
ANT_PARAMS="-Dfindbugs.dir=${REFINE_TOOLS_DIR}/${FINDBUGS_DIR}" ANT_PARAMS="-Dfindbugs.dir=${REFINE_TOOLS_DIR}/${FINDBUGS_DIR}"
ant findbugs ant findbugs
open "$REFINE_BUILD_DIR/reports/findbugs.html" display "$REFINE_BUILD_DIR/reports/findbugs.html"
} }
pmd() { pmd() {
@ -769,7 +791,7 @@ pmd() {
ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}" ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}"
ant pmd ant pmd
open "$REFINE_BUILD_DIR/reports/pmd.html" display "$REFINE_BUILD_DIR/reports/pmd.html"
} }
cpd() { cpd() {
@ -778,7 +800,7 @@ cpd() {
ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}" ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}"
ant cpd ant cpd
open "$REFINE_BUILD_DIR/reports/cpd.txt" display "$REFINE_BUILD_DIR/reports/cpd.txt"
} }
jslint() { jslint() {
@ -787,7 +809,7 @@ jslint() {
ANT_PARAMS="-Djslint.dir=${REFINE_TOOLS_DIR}/${JSLINT_DIR}" ANT_PARAMS="-Djslint.dir=${REFINE_TOOLS_DIR}/${JSLINT_DIR}"
ant jslint ant jslint
open "$REFINE_BUILD_DIR/reports/jslint.txt" display "$REFINE_BUILD_DIR/reports/jslint.txt"
} }
whitespace() { whitespace() {
@ -839,8 +861,9 @@ load_configs refine.ini
if [ "$OS" = "macosx" ] ; then if [ "$OS" = "macosx" ] ; then
if [ -z "$JAVA_HOME" ] ; then if [ -z "$JAVA_HOME" ] ; then
# Mac OS X defaults to Java 5. So update JAVA_HOME unless the user manually set it. # We need want recent Java because we're bundling JRE - may want to warn and force developer to set JAVA_HOME
export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home" # The /usr/libexec/java_home utility may be tied to the Java prefs app, so could go away when Apple removes it
export JAVA_HOME=$(/usr/libexec/java_home)
fi fi
fi fi
@ -869,6 +892,7 @@ while [ $# -ne 0 ] ; do
-w) shift; REFINE_WEBAPP="$1"; shift; continue;; -w) shift; REFINE_WEBAPP="$1"; shift; continue;;
-d) shift; REFINE_DATA_DIR="$1"; shift; continue;; -d) shift; REFINE_DATA_DIR="$1"; shift; continue;;
-m) shift; REFINE_MEMORY="$1"; shift; continue;; -m) shift; REFINE_MEMORY="$1"; shift; continue;;
-k) shift; REFINE_GOOGLE_API_KEY="$1"; shift; continue;;
-v) shift; REFINE_VERBOSITY="$1"; shift; continue;; -v) shift; REFINE_VERBOSITY="$1"; shift; continue;;
-x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;; -x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;;
--debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;; --debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
@ -953,7 +977,6 @@ add_option "-Dpython.path=$JYTHONPATH"
add_option "-Dpython.cachedir=$HOME/.local/share/google/refine/cachedir" add_option "-Dpython.cachedir=$HOME/.local/share/google/refine/cachedir"
# ----- Respond to the action given -------------------------------------------- # ----- Respond to the action given --------------------------------------------
case "$ACTION" in case "$ACTION" in
build) build_prepare; ant build;; build) build_prepare; ant build;;
clean) ant clean;; clean) ant clean;;

16
refine.bat
View File

@ -23,25 +23,25 @@ goto endUtils
echo Usage %0 [options] ^<action^> echo Usage %0 [options] ^<action^>
echo where [options] include: echo where [options] include:
echo. echo.
echo /? print this message and exit echo "/?" print this message and exit
echo. echo.
echo /p <port> the port that OpenRefine will listen to echo "/p <port>" the port that OpenRefine will listen to
echo default: 3333 echo default: 3333
echo. echo.
echo /i <interface> the host interface OpenRefine should bind to echo "/i <interface>" the host interface OpenRefine should bind to
echo default: 127.0.0.1 echo default: 127.0.0.1
echo. echo.
echo /w <path> path to the webapp echo "/w <path>" path to the webapp
echo default src\main\webapp echo default src\main\webapp
echo. echo.
echo /d enable JVM debugging (on port 8000) echo "/d" enable JVM debugging (on port 8000)
echo. echo.
echo /m <memory> max memory heap size to use echo "/m <memory>" max memory heap size to use
echo default: 1024M echo default: 1024M
echo. echo.
echo /x enable JMX monitoring (for jconsole and friends) echo "/x" enable JMX monitoring (for jconsole and friends)
echo. echo.
echo and <action> is one of echo "and <action> is one of
echo. echo.
echo build ..................... Build OpenRefine echo build ..................... Build OpenRefine
echo run ....................... Run OpenRefine echo run ....................... Run OpenRefine

2
refine.ini
View File

@ -5,7 +5,7 @@ no_proxy="localhost,127.0.0.1"
#REFINE_PORT=3334 #REFINE_PORT=3334
#REFINE_HOST=127.0.0.1 #REFINE_HOST=127.0.0.1
#REFINE_WEBAPP=main\webapp #REFINE_WEBAPP=main\webapp
#REFINE_MEMORY=1024M REFINE_MEMORY=3000M
# Some sample configurations. These have no defaults. # Some sample configurations. These have no defaults.
#ANT_HOME=C:\grefine\tools\apache-ant-1.8.1 #ANT_HOME=C:\grefine\tools\apache-ant-1.8.1