s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #6 from OpenRefine/master

Browse Source 
test
This commit is contained in:
Fabio 2013-10-15 07:34:17 -07:00
commit 6a442f6eed
171 changed files with 31582 additions and 4810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.classpath
View File

@ -11,15 +11,14 @@
<classpathentry exported="true" kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/ant-tools-1.8.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/arithcode-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-trunk-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/clojure-1.4.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/clojure-1.5.1-slim.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-codec-1.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/icu4j-4.2.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
@ -76,6 +75,13 @@
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-api-services-freebase-v1-rev25-1.13.2-beta.jar" sourcepath="extensions/freebase/module/MOD-INF/libsrc/google-api-services-freebase-v1-rev25-1.13.2-beta-sources.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar"/>
<classpathentry exported="true" kind="lib" path="extensions/freebase/module/MOD-INF/lib/mail.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-client-1.13.2-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v1-rev17-1.13.2-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-api-services-fusiontables-v1-rev17-1.13.2-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-http-client-1.13.1-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.13.1-beta.jar" sourcepath="extensions/gdata/module/MOD-INF/lib-src/google-oauth-client-1.13.1-beta-sources.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.13.1-beta.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jsr305-1.3.9.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.2.5.jar"/>

4
.gitignore vendored
View File

@ -1,5 +1,8 @@
*~
\#*#
*.DS_Store
*.class
.com.apple.timemachine.supported
.import-temp/
build/
dist/
@ -18,4 +21,5 @@ broker/core/module/MOD-INF/classes/
broker/core/WEB-INF/lib/
broker/core/data/
broker/core/test-output/
tmp/
/test-output

23
build.xml
View File

@ -14,7 +14,6 @@
<property environment="env"/>
<property name="version" value="trunk"/>
<property name="revision" value="rXXXX"/>
<property name="full_version" value="0.0.0.0"/>
<property name="build.dir" value="build"/>
<property name="dist.dir" value="dist"/>
@ -25,7 +24,7 @@
<property name="appengine.version" value="1"/>
<property name="appengine.sdk.dir" value="/opt/appengine"/>
<property name="fullname" value="openrefine-${version}-${revision}" />
<property name="fullname" value="openrefine-${version}" />
<property name="main.dir" value="${basedir}/main" />
@ -174,7 +173,7 @@
<taskdef resource="testngtasks" classpath="${server.tests.lib.dir}/testng-6.8.jar"/>
<mkdir dir="${build.dir}/server_tests"/>
<target name="server_test" depends="build_tests">
<testng workingdir="${build.dir}/server_tests"
<testng verbose="5" haltOnFailure="true" workingdir="${build.dir}/server_tests"
listener="org.testng.reporters.DotTestListener" excludedgroups="broken"
classpathref="tests.class.path">
<xmlfileset file="${server.tests.dir}/conf/tests.xml"/>
@ -207,6 +206,7 @@
<include name="**/*"/>
<exclude name="WEB-INF/classes/**"/>
<exclude name="WEB-INF/lib-src/**"/>
<exclude name="WEB-INF/lib/icu4j*.jar"/>
</fileset>
</copy>
@ -217,16 +217,17 @@
<include name="**/*"/>
<exclude name="**/build.xml"/>
<exclude name="**/src/**"/>
<exclude name="**/lib-src/**"/>
<exclude name="**/libsrc/**"/>
</fileset>
</copy>
<replace file="${built.webapp.dir}/WEB-INF/web.xml">
<replacefilter token="$VERSION" value="${version}"/>
<replacefilter token="$REVISION" value="${revision}"/>
</replace>
<replace file="${built.webapp.dir}/WEB-INF/butterfly.properties">
<replacefilter token="../../extensions/" value="extensions"/>
<replacefilter token="../../extensions" value="extensions"/>
</replace>
</target>
@ -254,14 +255,13 @@
</classpath>
<option value="-Xms256M"/>
<option value="-Xmx1024M"/>
<option value="-Drefine.version=${revision}"/>
<option value="-Drefine.version=${version}"/>
<option value="-Drefine.webapp=$APP_ROOT/Contents/Resource/${built.webapp.name}"/>
</bundleapp>
<copy todir="${mac.dir}/OpenRefine.app/Contents/Resource">
<fileset dir="${build.dir}" id="librarypathset" >
<include name="${built.webapp.name}/**/**" />
<exclude name="**/*.class" />
</fileset>
</copy>
@ -277,6 +277,7 @@
</target>
<target name="windows" depends="jar, prepare_webapp">
<echo message="Full version ${full_version} and version ${version}"/>
<mkdir dir="${windows.dir}"/>
<taskdef
name="launch4j"
@ -294,7 +295,7 @@
<cp>server/lib/*.jar</cp>
</classPath>
<jre minVersion="1.6.0" jdkPreference="preferJre" initialHeapSize="256" maxHeapSize="1024">
<opt>-Djava.library.path=server/lib/native/windows -Drefine.version=${revision}</opt>
<opt>-Djava.library.path=server/lib/native/windows </opt>
</jre>
<versionInfo
fileVersion="${full_version}"
@ -302,7 +303,7 @@
fileDescription="openrefine"
copyright="Copyright (c) 2013 OpenRefine contributors, 2010, Google, Inc."
productVersion="${full_version}"
txtProductVersion="${full_version}"
txtProductVersion="${version}"
productName="OpenRefine"
companyName="OpenRefine team"
internalName="openrefine"
@ -341,7 +342,7 @@
<copy file="${basedir}/LICENSE.txt" tofile="${windows.dir}/LICENSE.txt"/>
<mkdir dir="${dist.dir}"/>
<zip destfile="${dist.dir}/openrefine-${version}-${revision}.zip" basedir="${windows.dir}/.." includes="${release.name}/**"/>
<zip destfile="${dist.dir}/openrefine-win-${version}.zip" basedir="${windows.dir}/.." includes="${release.name}/**"/>
</target>
<target name="linux" depends="jar, prepare_webapp">
@ -372,7 +373,7 @@
<copy file="${basedir}/refine" tofile="${linux.dir}/refine"/>
<mkdir dir="${dist.dir}"/>
<tar longfile="gnu" compression="gzip" destfile="${dist.dir}/openrefine-${version}-${revision}.tar.gz">
<tar longfile="gnu" compression="gzip" destfile="${dist.dir}/openrefine-linux-${version}.tar.gz">
<tarfileset dir="${linux.dir}/.." filemode="755">
<include name="${release.name}/refine"/>
</tarfileset>

1
extensions/freebase/module/MOD-INF/controller.js
View File

@ -59,7 +59,6 @@ function init() {
RS.registerCommand(module, "import-qa-data", new Packages.com.google.refine.freebase.commands.ImportQADataCommand());
RS.registerCommand(module, "mqlread", new Packages.com.google.refine.freebase.commands.MQLReadCommand());
RS.registerCommand(module, "mqlwrite", new Packages.com.google.refine.freebase.commands.MQLWriteCommand());
RS.registerCommand(module, "load-language", new Packages.com.google.refine.freebase.commands.LoadLanguageCommand());
var OR = Packages.com.google.refine.operations.OperationRegistry;

240
extensions/freebase/module/langs/translation-default.json
View File

@ -1,115 +1,125 @@
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it.",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Set Freebase API Key",
"align-schema": "Align to Freebase's schemas...",
"load": "Load into Freebase...",
"browse-data-load": "Browse data load details...",
"import-qa": "Import QA data",
"add-columns": "Add columns from Freebase ...",
"warning-load": "You have not tried to load the data in this project into Freebase yet."
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}

250
extensions/freebase/module/langs/translation-en.json
View File

@ -1,125 +1,125 @@
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it.",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Set Freebase API Key",
"align-schema": "Align to Freebase's schemas...",
"load": "Load into Freebase...",
"browse-data-load": "Browse data load details...",
"import-qa": "Import QA data",
"add-columns": "Add columns from Freebase ...",
"warning-load": "You have not tried to load the data in this project into Freebase yet."
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}
{
"fb-schema-alignment": {
"close-confirm": "There are unsaved changes. Close anyway?",
"status-warning": "There are unsaved changes.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Search for a property or pick one below",
"search-property": "Search for a property",
"cell": "cell",
"cells": "cells",
"which-column": "Which column?",
"configure": "Configure...",
"which-topic": "Which topic?",
"what-value": "What value?",
"anonymous": "anonymous",
"add-property": "add property",
"anonymous-node": "Anonymous Node",
"freebase-topic": "Freebase Topic",
"value": "Value",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "text",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "date/time",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "The cell's content is used ...",
"specify-fb-topic": "to specify a Freebase topic, as reconciled",
"type-new-topics": "Type new topics as",
"literal-value": "as a literal value",
"literal-type": "Literal type",
"text-language": "Text language",
"key-namespace": "as a key in a namespace",
"namespace": "Namespace",
"generate-anonymous": "Generate an anonymous graph node",
"assign-type": "Assign a type to the node",
"use-existing-topic": "Use one existing Freebase topic",
"value-type": "Value type",
"language": "Language",
"use-literal-value": "Use a literal value",
"column-warning": "You must select at least one column",
"new-node-warning": "For creating a new graph node, you need to specify a type for it.",
"namespace-warning": "Please specify the namespace.",
"anonymous-node-warning": "For generating an anonymous graph node, you need to specify a type for it.",
"specify-topic-warning": "Please specify which existing Freebase topic to use",
"specify-value-warning": "Please specify the value to use"
},
"fb-interface": {
"dialog-header": "Align to Freebase's Schemas",
"body-text": "The schema alignment skeleton below specifies how your grid-shaped data will be transformed into graph-shaped data in Freebase's schemas.",
"find-more": "Find out more ...",
"skeleton": "Skeleton",
"mql-preview": "MQL-like Preview",
"tripleloader-preview": "TripleLoader Preview"
},
"fb-dialogs": {
"sign-in": "Sign into Freebase",
"enable-loading": "to enable loading",
"error-new-topic": "Error creating new topic",
"error-loading-data": "Error loading data",
"add-info-source": "Click here to add a new information source",
"dialog-header": "Load Data into Freebase",
"no-triples-dataset": "This dataset has no triples",
"warning-aligned": "Have you aligned it with Freebase's schemas yet?",
"name-of-data": "Name of data load",
"source-id": "Source ID (optional)",
"bodytext-1": "Note: Your data will only be loaded into",
"bodytext-2": "Sandbox is where everyone can experiment with Freebase technologies without disruption to the official",
"bodytext-3": "Sandbox gets",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " refreshed periodically",
"bodytext-4": "In order to load your data into the official Freebase, you must first load it into Sandbox. Then it must pass a Quality Assurance (QA) process before it can be loaded into Freebase proper.",
"quality-assurance": "Quality assurance",
"bodytext-5": "After loaded into Sandbox, enlist other people's help to double-check this data load's quality so that it can be loaded into Freebase.",
"triple-schedule": "triples successfully scheduled for loading",
"follow-progress": "Follow the loading progress in the ",
"refinery-link": "Freebase Refinery",
"signed-as": "Signed in as:",
"sign-out": "Sign Out"
},
"fb-qa": {
"header": "QA Data Load?",
"bodytext-1": "Other people will be enlisted to help double-check your data load for quality assurance purposes. Their time and labor have a cost.",
"bodytext-2": "You yourself should have taken all reasonable measures to eliminate errors from your data load. Your prudence is greatly appreciated.",
"tell-more": "Tell me more ...",
"ok-button": "Yes, QA Data Load"
},
"fb-extend": {
"add-column": "Add Columns from Freebase Based on Column",
"warning-add-properties": "Please add some properties first.",
"querying-freebase": "Querying Freebase ...",
"remove-column": "Remove this column",
"add-constraints": "Add constraints to this column",
"mql-constraints": "Enter MQL query constraints as JSON",
"warning-valid-json": "Please ensure that the JSON you enter is valid.",
"warning-json-obj": "The JSON you enter must be an object, that is, it is of this form { ... }.",
"add-property": "Add Property",
"suggested-properties": "Suggested Properties",
"constraint": "Constraint"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Set Freebase API Key",
"align-schema": "Align to Freebase's schemas...",
"load": "Load into Freebase...",
"browse-data-load": "Browse data load details...",
"import-qa": "Import QA data",
"add-columns": "Add columns from Freebase ...",
"warning-load": "You have not tried to load the data in this project into Freebase yet."
},
"fb-buttons": {
"save": "Save",
"save-load": "Save & Load",
"close": "Close",
"reset": "Reset",
"cancel": "Cancel",
"align-now": "Align Now",
"settings": "Settings",
"preview": "Preview",
"load-sandbox": "Load to Sandbox",
"ok": "Ok"
}
}

250
extensions/freebase/module/langs/translation-it.json
View File

@ -1,125 +1,125 @@
{
"fb-schema-alignment": {
"close-confirm": "Ci sono cambiamenti non salvati. Chiudere comunque?",
"status-warning": "Ci sono cambiamenti non salvati.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Cerca una proprietà o scegline una dalle seguenti",
"search-property": "Cerca una proprietà",
"cell": "cella",
"cells": "celle",
"which-column": "Quale colonna?",
"configure": "Configura...",
"which-topic": "Quale topic?",
"what-value": "Quale valore?",
"anonymous": "anonimo",
"add-property": "aggiungi proprietà",
"anonymous-node": "Nodo Anonimo",
"freebase-topic": "Topic Freebase",
"value": "Valore",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "testo",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "data/ora",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "Il contenuto della cella è usato ...",
"specify-fb-topic": "per specificare un topic Freebase, come riconciliato",
"type-new-topics": "Inserisci il nuovo topic come",
"literal-value": "come un valore letterale",
"literal-type": "Tipo letterale",
"text-language": "Lingua di testo",
"key-namespace": "come una chiave nel namespace",
"namespace": "Namespace",
"generate-anonymous": "Genera un nodo anonimo nel grafo",
"assign-type": "Assegna un tipo al nodo",
"use-existing-topic": "Usa un topic Freebase esistente",
"value-type": "Tipo valore",
"language": "Lingua",
"use-literal-value": "Usa un valore letterale",
"column-warning": "Devi selezionare almeno una colonna",
"new-node-warning": "Per creare un nuovo nodo nel grafo, devi specificarne un tipo.",
"namespace-warning": "Specifica il namespace.",
"anonymous-node-warning": "Per generare un nodo anonimo nel grafo, devi specificarne un tipo.",
"specify-topic-warning": "Per favore specifica quale topic esistente di Freebase usare",
"specify-value-warning": "Per favore specifica il valore da usare"
},
"fb-interface": {
"dialog-header": "Allinea con gli schemi Freebase",
"body-text": "Lo scheletro per l'allineamento della schema specifica come i tuoi dati in formato tabellare saranno trasformati in un formato a grafo con lo schema di Freebase.",
"find-more": "Per saperne di più ...",
"skeleton": "Scheletro",
"mql-preview": "Anteprima MQL-like",
"tripleloader-preview": "Anteprima TripleLoader"
},
"fb-dialogs": {
"sign-in": "Accedi a Freebase",
"enable-loading": "per abilitare il caricamento",
"error-new-topic": "Errore nella creazione di un nuovo topic",
"error-loading-data": "Errore durante il caricamento dei dati",
"add-info-source": "Clicca qui per aggiungere una nuova sorgente di informazioni",
"dialog-header": "Carica i dati in Freebase",
"no-triples-dataset": "Questo dataset non ha triple",
"warning-aligned": "Hai già effettuato l'allineamento con lo schema Freebase?",
"name-of-data": "Nome del caricamento dati",
"source-id": "ID sorgente (opzionale)",
"bodytext-1": "Nota: i tuoi dati saranno caricati solamente in",
"bodytext-2": "Sandbox è dove chiunque può sperimentare le tecnologie Freebase senza creare danni all'ufficiale",
"bodytext-3": "Sandbox viene",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " aggiornata periodicamente",
"bodytext-4": "Per caricare i tuoi data su Freebase ufficiale, devi prima caricarli nel Sandbox. Successivamente, devono passare un processo di analisi qualitativa prima di essere caricati su Freebase.",
"quality-assurance": "Analisi qualitativa",
"bodytext-5": "Dopo essere caricati nella Sandbox, viene chiesto l'aiuto di altre persone per controllare due volte la qualità di questi dati in modo da poter poi essere caricati in Freebase.",
"triple-schedule": "triple pianificate con successo per il caricamento",
"follow-progress": "Segui il processo di caricamento nel ",
"refinery-link": "Freebase Refinery",
"signed-as": "Accesso effettuato come:",
"sign-out": "Esci"
},
"fb-qa": {
"header": "Caricamento dati con analisi qualitativa?",
"bodytext-1": "Alcune persone verranno incaricate per controllare i tuoi dati allo scopo di verificarne la qualità. Il loro tempo e lavoro ha un costo.",
"bodytext-2": "Tu stesso dovresti aver già effettuato la maggior parte dei controlli per assicurarti di aver rimosso errori dai dati. La tua prudenza è enormemente apprezzata.",
"tell-more": "Dimmi di più ...",
"ok-button": "Si, Caricamento dati con analisi qualitativa"
},
"fb-extend": {
"add-column": "Aggiungi colonne da Freebase basandoti sulla colonna",
"warning-add-properties": "Prima aggiungi delle proprietà.",
"querying-freebase": "Interrogando Freebase ...",
"remove-column": "Rimuovi questa colonna",
"add-constraints": "Aggiungi vincoli a questa colonna",
"mql-constraints": "Inserisci i vincoli per la query MQL come JSON",
"warning-valid-json": "Assicurati che l'oggetto JSON inserito sia valido.",
"warning-json-obj": "Il JSON che inserisci deve essere un oggetto, cioè in questa forma: { ... }.",
"add-property": "Aggiungi Proprietà",
"suggested-properties": "Proprietà suggerite",
"constraint": "Vincoli"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Imposta l'API Key Freebase",
"align-schema": "Allinea con lo schema Freebase...",
"load": "Carica in Freebase...",
"browse-data-load": "Vedi i dettagli per il caricamento dati...",
"import-qa": "Importa dati con controllo qualità",
"add-columns": "Aggiungi colonne da Freebase ...",
"warning-load": "Non hai ancora provato a caricare i dati di questo progetto in Freebase."
},
"fb-buttons": {
"save": "Salva",
"save-load": "Salva & Carica",
"close": "Chiudi",
"reset": "Reset",
"cancel": "Cancella",
"align-now": "Allinea adesso",
"settings": "Settings",
"preview": "Anteprima",
"load-sandbox": "Carica nella Sandbox",
"ok": "Ok"
}
}
{
"fb-schema-alignment": {
"close-confirm": "Ci sono cambiamenti non salvati. Chiudere comunque?",
"status-warning": "Ci sono cambiamenti non salvati.",
"assert-link-found": "Assert link when 'true' is found in column",
"search-pick-property": "Cerca una proprietà o scegline una dalle seguenti",
"search-property": "Cerca una proprietà",
"cell": "cella",
"cells": "celle",
"which-column": "Quale colonna?",
"configure": "Configura...",
"which-topic": "Quale topic?",
"what-value": "Quale valore?",
"anonymous": "anonimo",
"add-property": "aggiungi proprietà",
"anonymous-node": "Nodo Anonimo",
"freebase-topic": "Topic Freebase",
"value": "Valore",
"skeleton-node": "Schema Alignment Skeleton Node",
"text": "testo",
"int": "int",
"float": "float",
"double": "double",
"boolean": "boolean",
"date-time": "data/ora",
"rawstring": "rawstring",
"set-to-cell": "Set to Cell in Column",
"cell-content-used": "Il contenuto della cella è usato ...",
"specify-fb-topic": "per specificare un topic Freebase, come riconciliato",
"type-new-topics": "Inserisci il nuovo topic come",
"literal-value": "come un valore letterale",
"literal-type": "Tipo letterale",
"text-language": "Lingua di testo",
"key-namespace": "come una chiave nel namespace",
"namespace": "Namespace",
"generate-anonymous": "Genera un nodo anonimo nel grafo",
"assign-type": "Assegna un tipo al nodo",
"use-existing-topic": "Usa un topic Freebase esistente",
"value-type": "Tipo valore",
"language": "Lingua",
"use-literal-value": "Usa un valore letterale",
"column-warning": "Devi selezionare almeno una colonna",
"new-node-warning": "Per creare un nuovo nodo nel grafo, devi specificarne un tipo.",
"namespace-warning": "Specifica il namespace.",
"anonymous-node-warning": "Per generare un nodo anonimo nel grafo, devi specificarne un tipo.",
"specify-topic-warning": "Per favore specifica quale topic esistente di Freebase usare",
"specify-value-warning": "Per favore specifica il valore da usare"
},
"fb-interface": {
"dialog-header": "Allinea con gli schemi Freebase",
"body-text": "Lo scheletro per l'allineamento della schema specifica come i tuoi dati in formato tabellare saranno trasformati in un formato a grafo con lo schema di Freebase.",
"find-more": "Per saperne di più ...",
"skeleton": "Scheletro",
"mql-preview": "Anteprima MQL-like",
"tripleloader-preview": "Anteprima TripleLoader"
},
"fb-dialogs": {
"sign-in": "Accedi a Freebase",
"enable-loading": "per abilitare il caricamento",
"error-new-topic": "Errore nella creazione di un nuovo topic",
"error-loading-data": "Errore durante il caricamento dei dati",
"add-info-source": "Clicca qui per aggiungere una nuova sorgente di informazioni",
"dialog-header": "Carica i dati in Freebase",
"no-triples-dataset": "Questo dataset non ha triple",
"warning-aligned": "Hai già effettuato l'allineamento con lo schema Freebase?",
"name-of-data": "Nome del caricamento dati",
"source-id": "ID sorgente (opzionale)",
"bodytext-1": "Nota: i tuoi dati saranno caricati solamente in",
"bodytext-2": "Sandbox è dove chiunque può sperimentare le tecnologie Freebase senza creare danni all'ufficiale",
"bodytext-3": "Sandbox viene",
"sandbox-link": " Sandbox",
"freebase-link": " Freebase",
"refreshed-link": " aggiornata periodicamente",
"bodytext-4": "Per caricare i tuoi data su Freebase ufficiale, devi prima caricarli nel Sandbox. Successivamente, devono passare un processo di analisi qualitativa prima di essere caricati su Freebase.",
"quality-assurance": "Analisi qualitativa",
"bodytext-5": "Dopo essere caricati nella Sandbox, viene chiesto l'aiuto di altre persone per controllare due volte la qualità di questi dati in modo da poter poi essere caricati in Freebase.",
"triple-schedule": "triple pianificate con successo per il caricamento",
"follow-progress": "Segui il processo di caricamento nel ",
"refinery-link": "Freebase Refinery",
"signed-as": "Accesso effettuato come:",
"sign-out": "Esci"
},
"fb-qa": {
"header": "Caricamento dati con analisi qualitativa?",
"bodytext-1": "Alcune persone verranno incaricate per controllare i tuoi dati allo scopo di verificarne la qualità. Il loro tempo e lavoro ha un costo.",
"bodytext-2": "Tu stesso dovresti aver già effettuato la maggior parte dei controlli per assicurarti di aver rimosso errori dai dati. La tua prudenza è enormemente apprezzata.",
"tell-more": "Dimmi di più ...",
"ok-button": "Si, Caricamento dati con analisi qualitativa"
},
"fb-extend": {
"add-column": "Aggiungi colonne da Freebase basandoti sulla colonna",
"warning-add-properties": "Prima aggiungi delle proprietà.",
"querying-freebase": "Interrogando Freebase ...",
"remove-column": "Rimuovi questa colonna",
"add-constraints": "Aggiungi vincoli a questa colonna",
"mql-constraints": "Inserisci i vincoli per la query MQL come JSON",
"warning-valid-json": "Assicurati che l'oggetto JSON inserito sia valido.",
"warning-json-obj": "Il JSON che inserisci deve essere un oggetto, cioè in questa forma: { ... }.",
"add-property": "Aggiungi Proprietà",
"suggested-properties": "Proprietà suggerite",
"constraint": "Vincoli"
},
"fb-menu": {
"freebase": "Freebase",
"set-api-key": "Imposta l'API Key Freebase",
"align-schema": "Allinea con lo schema Freebase...",
"load": "Carica in Freebase...",
"browse-data-load": "Vedi i dettagli per il caricamento dati...",
"import-qa": "Importa dati con controllo qualità",
"add-columns": "Aggiungi colonne da Freebase ...",
"warning-load": "Non hai ancora provato a caricare i dati di questo progetto in Freebase."
},
"fb-buttons": {
"save": "Salva",
"save-load": "Salva & Carica",
"close": "Chiudi",
"reset": "Reset",
"cancel": "Cancella",
"align-now": "Allinea adesso",
"settings": "Settings",
"preview": "Anteprima",
"load-sandbox": "Carica nella Sandbox",
"ok": "Ok"
}
}

2
extensions/freebase/module/scripts/dialogs/schema-alignment/schema-alignment-dialog.html
View File

@ -2,7 +2,7 @@
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<p class="body-text" bind="body_text">
<a href="http://github.com/OpenRefine/OpenRefine/wiki/SchemaAlignment" target="_blank" bind="find_more"></a>
<a href="https://github.com/OpenRefine/OpenRefine/wiki/Schema-Alignment" target="_blank" bind="find_more"></a>
</p>
<div id="schema-alignment-tabs" class="refine-tabs">
<ul>

2
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
View File

@ -471,7 +471,7 @@ SchemaAlignmentDialog.UINode.prototype._showNodeConfigDialog = function() {
});
if (column.name in columnMap) {
radio.attr("checked", "true");
radio.prop("checked", true);
}
$('<span></span>').text(column.name).appendTo(tr.insertCell(1));

5
extensions/freebase/module/scripts/extension.js
View File

@ -39,11 +39,12 @@ var lang = navigator.language.split("-")[0]
|| navigator.userLanguage.split("-")[0];
var dictionary = "";
$.ajax({
url : "/command/freebase/load-language?",
url : "/command/core/load-language?",
type : "POST",
async : false,
data : {
lng : lang
module : "freebase",
// lang : lang
},
success : function(data) {
dictionary = data;

73
extensions/freebase/src/com/google/refine/freebase/commands/LoadLanguageCommand.java
View File

@ -1,73 +0,0 @@
package com.google.refine.freebase.commands;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
public class LoadLanguageCommand extends Command {
public LoadLanguageCommand() {
// TODO Auto-generated constructor stub
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String rawDirectoryFile = request.getSession().getServletContext()
.getRealPath("extensions/freebase/module/langs/");
String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
BufferedReader reader = null;String param = null;
try {
param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
} catch (NullPointerException e) {
}
if (param == null) param = request.getParameter("lng");
String[] langs = param.split(" ");
try {
String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e1) {
try {
String file = cleanedDirectory + File.separator + "translation-default.json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e3) {
e3.printStackTrace();
}
}
String line = null;
String message = new String();
if (reader != null) {
while ((line = reader.readLine()) != null) {
// buffer.append(line);
message += line + System.getProperty("line.separator");
}
}
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.getWriter().println(message);
response.getWriter().flush();
response.getWriter().close();
}
}

3
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
View File

@ -430,6 +430,9 @@ public class FreebaseUtils {
/**
* This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
*
* NOTE: JSONRPC has been deprecated and replaced by HTTP Batch (which also
* doesn't support MQLread, so perhaps we should just remove this))
*/
@SuppressWarnings("unused")
static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {

4
extensions/gdata/build.xml
View File

@ -63,7 +63,9 @@
<copy todir="${ext.dir}/module">
<fileset dir="module">
<include name="**/*.*"/>
<include name="**/*"/>
<exclude name="**/lib-src/*"/>
<exclude name="**/src/*"/>
</fileset>
</copy>

27
extensions/gdata/module/MOD-INF/controller.js
View File

@ -31,6 +31,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Controller for GData extension.
*
* This is run in the Butterfly (ie Refine) server context using the Rhino
* Javascript interpreter.
*/
var html = "text/html";
var encoding = "UTF-8";
var version = "0.2";
@ -43,7 +50,7 @@ function init() {
var RS = Packages.com.google.refine.RefineServlet;
RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand());
RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand());
RS.registerCommand(module, "load-language", Packages.com.google.refine.extension.gdata.commands.LoadLanguageCommand());
// TODO: Need a new OAUTH2 authorize command for FusionTables
// Register importer and exporter
var IM = Packages.com.google.refine.importing.ImportingManager;
@ -101,21 +108,13 @@ function process(path, request, response) {
} else if (path == "authorized") {
var context = {};
context.winname = request.getParameter("winname");
context.callback = request.getParameter("callback");
context.callback = request.getParameter("cb");
(function() {
var queryString = request.getQueryString();
if (queryString != null) {
var AuthSubUtil = Packages.com.google.gdata.client.http.AuthSubUtil;
// FIXME(SM): can we safely assume UTF-8 encoding here?
var onetimeUseToken = AuthSubUtil.getTokenFromReply(
Packages.java.net.URLDecoder.decode(queryString, "UTF-8"));
if (onetimeUseToken) {
var sessionToken = AuthSubUtil.exchangeForSessionToken(onetimeUseToken, null);
Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, sessionToken);
return;
}
var token = Packages.com.google.refine.extension.gdata.GDataExtension.getTokenFromCode(module,request);
if (token) {
Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, token);
return;
}
Packages.com.google.refine.extension.gdata.TokenCookie.deleteToken(request, response);
})();

BIN
extensions/gdata/module/MOD-INF/lib-src/google-api-client-1.13.2-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib-src/google-api-services-fusiontables-v1-rev17-1.13.2-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib-src/google-http-client-1.13.1-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib-src/google-http-client-jackson-1.13.1-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib-src/google-oauth-client-1.13.1-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib-src/google-oauth-client-servlet-1.13.1-beta-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-api-client-1.13.2-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v1-rev17-1.13.2-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-http-client-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.13.1-beta.jar Normal file
View File

Binary file not shown.

BIN
extensions/gdata/module/MOD-INF/lib/jsr305-1.3.9.jar Normal file
View File

Binary file not shown.

112
extensions/gdata/module/langs/translation-default.json
View File

@ -1,56 +1,56 @@
{
"gdata-import": {
"preparing": "Preparing ...",
"creating": "Creating project ...",
"title": "Public Documents",
"import-by-url": "Import a <em>public</em> Google Spreadsheet or Fusion Table by its URL:",
"next->": "Next &raquo;",
"auth-doc": "Authorized Documents",
"please": "Please",
"sign-in": "sign in and authorize",
"sign-out": "sign out",
"access-data": "access to your Google data.",
"retrieving": "Retrieving Google Docs documents ...",
"re-sign-in": "re-sign in",
"another-account": "with another account"
},
"gdata-parsing": {
"start-over": "&laquo; Start Over",
"conf-pars": "Configure Parsing Options",
"proj-name": "Project&nbsp;name",
"create-proj": "Create Project &raquo;",
"updating-preview": "Updating preview ...",
"worksheet": "Worksheets",
"option": "Options",
"preview-button": "Update&nbsp;Preview",
"ignore-first": "Ignore first",
"ignore": "line(s) at beginning of file",
"parse-next": "Parse next",
"parse": "line(s) as column headers",
"discard-next": "Discard initial",
"discard": "row(s) of data",
"limit-next": "Load at most",
"limit": "row(s) of data",
"store-row": "Store blank rows",
"store-cell": "Store blank cells as nulls"
},
"gdata-source": {
"alert-url": "You must specify a web address (URL) to import.",
"type": "Type",
"title": "Title",
"authors": "Authors",
"updated": "Updated"
},
"gdata-exporter": {
"uploading": "Uploading...",
"upload-error": "Upload error: ",
"new-spreadsheet": "A new Google spreadsheet",
"enter-spreadsheet": "Enter a name for the new Google spreadsheet",
"new-fusion": "A new Google Fusion table",
"enter-fusion": "Enter a name for the new Google Fusion table"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Authorization",
"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
}
}
{
"gdata-import": {
"preparing": "Preparing ...",
"creating": "Creating project ...",
"title": "Public Documents",
"import-by-url": "Import a <em>public</em> Google Spreadsheet or Fusion Table by its URL:",
"next->": "Next &raquo;",
"auth-doc": "Authorized Documents",
"please": "Please",
"sign-in": "sign in and authorize",
"sign-out": "sign out",
"access-data": "access to your Google data.",
"retrieving": "Retrieving Google Docs documents ...",
"re-sign-in": "re-sign in",
"another-account": "with another account"
},
"gdata-parsing": {
"start-over": "&laquo; Start Over",
"conf-pars": "Configure Parsing Options",
"proj-name": "Project&nbsp;name",
"create-proj": "Create Project &raquo;",
"updating-preview": "Updating preview ...",
"worksheet": "Worksheets",
"option": "Options",
"preview-button": "Update&nbsp;Preview",
"ignore-first": "Ignore first",
"ignore": "line(s) at beginning of file",
"parse-next": "Parse next",
"parse": "line(s) as column headers",
"discard-next": "Discard initial",
"discard": "row(s) of data",
"limit-next": "Load at most",
"limit": "row(s) of data",
"store-row": "Store blank rows",
"store-cell": "Store blank cells as nulls"
},
"gdata-source": {
"alert-url": "You must specify a web address (URL) to import.",
"type": "Type",
"title": "Title",
"authors": "Authors",
"updated": "Updated"
},
"gdata-exporter": {
"uploading": "Uploading...",
"upload-error": "Upload error: ",
"new-spreadsheet": "A new Google spreadsheet",
"enter-spreadsheet": "Enter a name for the new Google spreadsheet",
"new-fusion": "A new Google Fusion table",
"enter-fusion": "Enter a name for the new Google Fusion table"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Authorization",
"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
}
}

112
extensions/gdata/module/langs/translation-en.json
View File

@ -1,56 +1,56 @@
{
"gdata-import": {
"preparing": "Preparing ...",
"creating": "Creating project ...",
"title": "Public Documents",
"import-by-url": "Import a <em>public</em> Google Spreadsheet or Fusion Table by its URL:",
"next->": "Next &raquo;",
"auth-doc": "Authorized Documents",
"please": "Please",
"sign-in": "sign in and authorize",
"sign-out": "sign out",
"access-data": "access to your Google data.",
"retrieving": "Retrieving Google Docs documents ...",
"re-sign-in": "re-sign in",
"another-account": "with another account"
},
"gdata-parsing": {
"start-over": "&laquo; Start Over",
"conf-pars": "Configure Parsing Options",
"proj-name": "Project&nbsp;name",
"create-proj": "Create Project &raquo;",
"updating-preview": "Updating preview ...",
"worksheet": "Worksheets",
"option": "Options",
"preview-button": "Update&nbsp;Preview",
"ignore-first": "Ignore first",
"ignore": "line(s) at beginning of file",
"parse-next": "Parse next",
"parse": "line(s) as column headers",
"discard-next": "Discard initial",
"discard": "row(s) of data",
"limit-next": "Load at most",
"limit": "row(s) of data",
"store-row": "Store blank rows",
"store-cell": "Store blank cells as nulls"
},
"gdata-source": {
"alert-url": "You must specify a web address (URL) to import.",
"type": "Type",
"title": "Title",
"authors": "Authors",
"updated": "Updated"
},
"gdata-exporter": {
"uploading": "Uploading...",
"upload-error": "Upload error: ",
"new-spreadsheet": "A new Google spreadsheet",
"enter-spreadsheet": "Enter a name for the new Google spreadsheet",
"new-fusion": "A new Google Fusion table",
"enter-fusion": "Enter a name for the new Google Fusion table"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Authorization",
"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
}
}
{
"gdata-import": {
"preparing": "Preparing ...",
"creating": "Creating project ...",
"title": "Public Documents",
"import-by-url": "Import a <em>public</em> Google Spreadsheet or Fusion Table by its URL:",
"next->": "Next &raquo;",
"auth-doc": "Authorized Documents",
"please": "Please",
"sign-in": "sign in and authorize",
"sign-out": "sign out",
"access-data": "access to your Google data.",
"retrieving": "Retrieving Google Docs documents ...",
"re-sign-in": "re-sign in",
"another-account": "with another account"
},
"gdata-parsing": {
"start-over": "&laquo; Start Over",
"conf-pars": "Configure Parsing Options",
"proj-name": "Project&nbsp;name",
"create-proj": "Create Project &raquo;",
"updating-preview": "Updating preview ...",
"worksheet": "Worksheets",
"option": "Options",
"preview-button": "Update&nbsp;Preview",
"ignore-first": "Ignore first",
"ignore": "line(s) at beginning of file",
"parse-next": "Parse next",
"parse": "line(s) as column headers",
"discard-next": "Discard initial",
"discard": "row(s) of data",
"limit-next": "Load at most",
"limit": "row(s) of data",
"store-row": "Store blank rows",
"store-cell": "Store blank cells as nulls"
},
"gdata-source": {
"alert-url": "You must specify a web address (URL) to import.",
"type": "Type",
"title": "Title",
"authors": "Authors",
"updated": "Updated"
},
"gdata-exporter": {
"uploading": "Uploading...",
"upload-error": "Upload error: ",
"new-spreadsheet": "A new Google spreadsheet",
"enter-spreadsheet": "Enter a name for the new Google spreadsheet",
"new-fusion": "A new Google Fusion table",
"enter-fusion": "Enter a name for the new Google Fusion table"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Authorization",
"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
}
}

112
extensions/gdata/module/langs/translation-it.json
View File

@ -1,56 +1,56 @@
{
"gdata-import": {
"preparing": "In preparazione ...",
"creating": "Creazione il progetto ...",
"title": "Documenti Pubblici",
"import-by-url": "Importa un Google Spreadsheet o Fusion Table <em>pubblico</em> inserendo l'URL:",
"next->": "Avanti &raquo;",
"auth-doc": "Documenti Autorizzati",
"please": "Per piacere",
"sign-in": "autenticati ed autorizza",
"sign-out": "esci",
"access-data": "accedi ai tuoi dati Google.",
"retrieving": "Recuperando i documenti Google Doc ...",
"re-sign-in": "rieffettua l'autenticazione",
"another-account": "con un altro account"
},
"gdata-parsing": {
"start-over": "&laquo; Ricomincia",
"conf-pars": "Configura le opzioni per il parsing",
"proj-name": "Nome&nbsp;del&nbsp;progetto",
"create-proj": "Crea un progetto &raquo;",
"updating-preview": "Aggiornando la preview ...",
"worksheet": "Worksheets",
"option": "Opzioni",
"preview-button": "Aggiorna&nbsp;la&nbsp;Preview",
"ignore-first": "Ignora le prime",
"ignore": "linee all'inizio del file",
"parse-next": "Parsa le prossime",
"parse": "linee come nomi delle colonne",
"discard-next": "Scarta le prime",
"discard": "righe di dati",
"limit-next": "Carica al massimo",
"limit": "righe di dati",
"store-row": "Salva righe vuote",
"store-cell": "Salva le celle vuote come 'null'"
},
"gdata-source": {
"alert-url": "Devi specificare un indirizzo web (URL) per l'import.",
"type": "Tipo",
"title": "Titolo",
"authors": "Autori",
"updated": "Aggiornato"
},
"gdata-exporter": {
"uploading": "Caricando...",
"upload-error": "Errore durante il caricamento: ",
"new-spreadsheet": "Un nuovo Google spreadsheet",
"enter-spreadsheet": "Inserisci un nome per il nuovo Google spreadsheet",
"new-fusion": "Una nuova tabella Google Fusion",
"enter-fusion": "Inserisci un nome per la nuova tabella Google Fusion"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Autorizzazione",
"authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine."
}
}
{
"gdata-import": {
"preparing": "In preparazione ...",
"creating": "Creazione il progetto ...",
"title": "Documenti Pubblici",
"import-by-url": "Importa un Google Spreadsheet o Fusion Table <em>pubblico</em> inserendo l'URL:",
"next->": "Avanti &raquo;",
"auth-doc": "Documenti Autorizzati",
"please": "Per piacere",
"sign-in": "autenticati ed autorizza",
"sign-out": "esci",
"access-data": "accedi ai tuoi dati Google.",
"retrieving": "Recuperando i documenti Google Doc ...",
"re-sign-in": "rieffettua l'autenticazione",
"another-account": "con un altro account"
},
"gdata-parsing": {
"start-over": "&laquo; Ricomincia",
"conf-pars": "Configura le opzioni per il parsing",
"proj-name": "Nome&nbsp;del&nbsp;progetto",
"create-proj": "Crea un progetto &raquo;",
"updating-preview": "Aggiornando la preview ...",
"worksheet": "Worksheets",
"option": "Opzioni",
"preview-button": "Aggiorna&nbsp;la&nbsp;Preview",
"ignore-first": "Ignora le prime",
"ignore": "linee all'inizio del file",
"parse-next": "Parsa le prossime",
"parse": "linee come nomi delle colonne",
"discard-next": "Scarta le prime",
"discard": "righe di dati",
"limit-next": "Carica al massimo",
"limit": "righe di dati",
"store-row": "Salva righe vuote",
"store-cell": "Salva le celle vuote come 'null'"
},
"gdata-source": {
"alert-url": "Devi specificare un indirizzo web (URL) per l'import.",
"type": "Tipo",
"title": "Titolo",
"authors": "Autori",
"updated": "Aggiornato"
},
"gdata-exporter": {
"uploading": "Caricando...",
"upload-error": "Errore durante il caricamento: ",
"new-spreadsheet": "Un nuovo Google spreadsheet",
"enter-spreadsheet": "Inserisci un nome per il nuovo Google spreadsheet",
"new-fusion": "Una nuova tabella Google Fusion",
"enter-fusion": "Inserisci un nome per la nuova tabella Google Fusion"
},
"gdata-auth": {
"authorize-label": "OpenRefine - Autorizzazione",
"authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine."
}
}

4
extensions/gdata/module/scripts/gdata-extension.js
View File

@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
var GdataExtension = {};
GdataExtension.isAuthorized = function() {
return $.cookie('authsub_token') !== null;
return $.cookie('oauth2_token') !== null;
};
GdataExtension.showAuthorizationDialog = function(onAuthorized, onNotAuthorized) {
@ -57,6 +57,6 @@ GdataExtension.showAuthorizationDialog = function(onAuthorized, onNotAuthorized)
};
window[callbackName] = callback;
var url = ModuleWirings['gdata'] + "authorize?winname=" + escape(windowName) + "&callback=" + escape(callbackName);
var url = ModuleWirings['gdata'] + "authorize?winname=" + escape(windowName) + "&cb=" + escape(callbackName);
var win = window.open(url, "openrefinegdataauth", "resizable=1,width=800,height=600");
};

4
extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html
View File

@ -9,12 +9,12 @@
<div bind="dataPanel" class="gdata-importing-parsing-data-panel"></div>
<div bind="progressPanel" class="gdata-importing-progress-data-panel">
<img src="images/large-spinner.gif" /> <span bind="gdata-updating"></span>
<img src="images/large-spinner.gif" /> <span bind="gdata_updating"></span>
</div>
<div bind="controlPanel" class="gdata-importing-parsing-control-panel"><div class="grid-layout layout-normal"><table>
<tr>
<td bind="gdata-options"></td>
<td bind="gdata_options"></td>
<td><button class="button" bind="previewButton"></button></td>
</tr>
<tr>

6
extensions/gdata/module/scripts/index/gdata-parsing-panel.html
View File

@ -9,13 +9,13 @@
<div bind="dataPanel" class="gdata-importing-parsing-data-panel"></div>
<div bind="progressPanel" class="gdata-importing-progress-data-panel">
<img src="images/large-spinner.gif" /> <span bind="gdata-updating"></span>
<img src="images/large-spinner.gif" /> <span bind="gdata_updating"></span>
</div>
<div bind="controlPanel" class="gdata-importing-parsing-control-panel"><div class="grid-layout layout-normal"><table>
<tr>
<td bind="gdata-worksheet"></td>
<td colspan="2" bind="gdata-options"></td>
<td bind="gdata_worksheet"></td>
<td colspan="2" bind="gdata_options"></td>
<td rowspan="2"><button class="button" bind="previewButton"></button></td>
</tr>
<tr>

4
extensions/gdata/module/scripts/index/gdata-source-ui.js
View File

@ -49,7 +49,7 @@ Refine.GDataSourceUI.prototype.attachUI = function(body) {
$('#gdata-signin-btn').text($.i18n._("gdata-import")["sign-in"]);
$('#gdata-access-data').text($.i18n._("gdata-import")["access-data"]);
$('#gdata-retrieving').text($.i18n._("gdata-import")["retrieving"]);
$('#gdata-retrieving').text($.i18n._("gdata-import")["sign-out"]);
$('#gdata-signout').text($.i18n._("gdata-import")["sign-out"]);
$('#gdata-resignin').text($.i18n._("gdata-import")["re-sign-in"]);
$('#gdata-another-account').text($.i18n._("gdata-import")["another-account"]);
@ -143,7 +143,7 @@ Refine.GDataSourceUI.prototype._renderDocuments = function(o) {
$('<span>').text(doc.type).appendTo(td);
td = tr.insertCell(tr.cells.length);
var title = $('<a>')
$('<a>')
.addClass('gdata-doc-title')
.attr('href', 'javascript:{}')
.text(doc.title)

27
extensions/gdata/module/scripts/index/importing-controller.js
View File

@ -36,11 +36,12 @@ var lang = navigator.language.split("-")[0]
|| navigator.userLanguage.split("-")[0];
var dictionary = "";
$.ajax({
url : "/command/gdata/load-language?",
url : "/command/core/load-language?",
type : "POST",
async : false,
data : {
lng : lang
module : "gdata",
// lang : lang
},
success : function(data) {
dictionary = data;
@ -165,7 +166,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
this._parsingPanelElmts = DOM.bind(this._parsingPanel);
if(this._doc.type != 'table'){
this._parsingPanelElmts.gdata-worksheet.html($.i18n._('gdata-parsing')["worksheet"]);
this._parsingPanelElmts.gdata_worksheet.html($.i18n._('gdata-parsing')["worksheet"]);
this._parsingPanelElmts.gdata_ignore_first.html($.i18n._('gdata-parsing')["ignore-first"]);
this._parsingPanelElmts.gdata_ignore.html($.i18n._('gdata-parsing')["ignore"]);
this._parsingPanelElmts.gdata_parse_next.html($.i18n._('gdata-parsing')["parse-next"]);
@ -175,9 +176,9 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
this._parsingPanelElmts.gdata_conf_pars.html($.i18n._('gdata-parsing')["conf-pars"]);
this._parsingPanelElmts.gdata_proj_name.html($.i18n._('gdata-parsing')["proj-name"]);
this._parsingPanelElmts.createProjectButton.html($.i18n._('gdata-parsing')["create-proj"]);
this._parsingPanelElmts.gdata-options.html($.i18n._('gdata-parsing')["option"]);
this._parsingPanelElmts.gdata_options.html($.i18n._('gdata-parsing')["option"]);
this._parsingPanelElmts.previewButton.html($.i18n._('gdata-parsing')["preview-button"]);
this._parsingPanelElmts.gdata-updating.html($.i18n._('gdata-parsing')["updating-preview"]);
this._parsingPanelElmts.gdata_updating.html($.i18n._('gdata-parsing')["updating-preview"]);
this._parsingPanelElmts.gdata_discard_next.html($.i18n._('gdata-parsing')["discard-next"]);
this._parsingPanelElmts.gdata_discard.html($.i18n._('gdata-parsing')["discard"]);
this._parsingPanelElmts.gdata_limit_next.html($.i18n._('gdata-parsing')["limit-next"]);
@ -234,7 +235,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
if (this._doc.type != 'table') {
var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0];
$.each(this._options.worksheets, function(i, v) {
var id = 'gdata-worksheet-' + Math.round(Math.random() * 1000000);
var id = 'gdata_worksheet-' + Math.round(Math.random() * 1000000);
var tr = sheetTable.insertRow(sheetTable.rows.length);
var td0 = $(tr.insertCell(0)).attr('width', '1%');
@ -245,7 +246,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
.attr('sheetUrl', this.link)
.appendTo(td0);
if (i === 0) {
checkbox.attr('checked', 'true');
checkbox.prop("checked", true);
}
$('<label>')
@ -260,28 +261,28 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
});
if (this._options.ignoreLines > 0) {
this._parsingPanelElmts.ignoreCheckbox.attr("checked", "checked");
this._parsingPanelElmts.ignoreCheckbox.prop("checked", true);
this._parsingPanelElmts.ignoreInput[0].value = this._options.ignoreLines.toString();
}
if (this._options.headerLines > 0) {
this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked");
this._parsingPanelElmts.headerLinesCheckbox.prop("checked", true);
this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString();
}
}
if (this._options.limit > 0) {
this._parsingPanelElmts.limitCheckbox.attr("checked", "checked");
this._parsingPanelElmts.limitCheckbox.prop("checked", true);
this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString();
}
if (this._options.skipDataLines > 0) {
this._parsingPanelElmts.skipCheckbox.attr("checked", "checked");
this._parsingPanelElmts.skipCheckbox.prop("checked", true);
this._parsingPanelElmts.skipInput.value[0].value = this._options.skipDataLines.toString();
}
if (this._options.storeBlankRows) {
this._parsingPanelElmts.storeBlankRowsCheckbox.attr("checked", "checked");
this._parsingPanelElmts.storeBlankRowsCheckbox.prop("checked", true);
}
if (this._options.storeBlankCellsAsNulls) {
this._parsingPanelElmts.storeBlankCellsAsNullsCheckbox.attr("checked", "checked");
this._parsingPanelElmts.storeBlankCellsAsNullsCheckbox.prop("checked", true);
}
var onChange = function() {

5
extensions/gdata/module/scripts/project/exporters.js
View File

@ -33,11 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
var dictionary = "";
$.ajax({
url : "/command/gdata/load-language?",
url : "/command/core/load-language?",
type : "POST",
async : false,
data : {
lng : lang
module : "gdata",
// lang : lang
},
success : function(data) {
dictionary = data;

19
extensions/gdata/src/com/google/refine/extension/gdata/DeAuthorizeCommand.java
View File

@ -34,12 +34,18 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.gdata.client.http.AuthSubUtil;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.refine.commands.Command;
public class DeAuthorizeCommand extends Command {
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -50,7 +56,16 @@ public class DeAuthorizeCommand extends Command {
String sessionToken = TokenCookie.getToken(request);
if (sessionToken != null) {
AuthSubUtil.revokeToken(sessionToken, null);
// No method to do this in Google's client lib, so roll our own
HttpRequestFactory factory = HTTP_TRANSPORT.createRequestFactory();
GenericUrl url = new GenericUrl("https://accounts.google.com/o/oauth2/revoke?token="+sessionToken);
HttpRequest rqst = factory.buildGetRequest(url);
HttpResponse resp = rqst.execute();
if (resp.getStatusCode() != 200) {
respond(response, String.valueOf(resp.getStatusCode()), resp.getStatusMessage());
}
TokenCookie.deleteToken(request, response);
}
respond(response, "200 OK", "");

166
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java Normal file
View File

@ -0,0 +1,166 @@
/*
* Copyright (c) 2013, Thomas F. Morris and other contributors
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.Fusiontables.Query.Sql;
import com.google.api.services.fusiontables.Fusiontables.Query.SqlGet;
import com.google.api.services.fusiontables.Fusiontables.Table.ImportRows;
import com.google.api.services.fusiontables.model.FusiontablesImport;
import com.google.api.services.fusiontables.model.Sqlresponse;
/**
* @author Tom Morris <tfmorris@gmail.com>
* @copyright 2010,2013 Thomas F. Morris
*/
public class FusionTableHandler {
static private Sqlresponse executeQuery (Fusiontables service, String query)
throws IOException {
Sql sql = service.query().sql(query);
Sqlresponse response = sql.execute();
return response;
}
static String createTable(Fusiontables service, String name, List<String> columnNames) throws IOException {
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE '");
sb.append(name);
sb.append("' (");
boolean first = true;
for (String columnName : columnNames) {
if (first) {
first = false;
} else {
sb.append(',');
}
sb.append("'");
sb.append(columnName);
sb.append("': STRING");
}
sb.append(")");
String createQuery = sb.toString();
Sqlresponse response = executeQuery(service, createQuery);
// response.getTableId(); // FIXME: Oh wait, there's no such F*ing method!!!
return getTableId(response);
}
private static String getTableId(Sqlresponse response) {
List<Object> row = response.getRows().get(0);
int i = 0;
for (String colname : response.getColumns()) {
if ("tableid".equals(colname)) {
return (String) row.get(i);
}
}
return null;
}
/**
* Insert a set of rows and optionally return the IDs of the new rows.
*
* @param service a Fusiontables object
* @param sql SQL statement to do the inserts
* @param returnIds true to return the IDs of the newly inserted rows
* @return
* @throws IOException
*/
static Long insertRows(Fusiontables service, String tableId, AbstractInputStreamContent mediaContent) throws IOException {
ImportRows importRows = service.table().importRows(tableId, mediaContent);
importRows.setIsStrict(false);
FusiontablesImport response = importRows.execute();
return response.getNumRowsReceived();
}
static String getFusionTableKey(URL url) {
String tableId = getParamValue(url,"dsrcid"); // old style phased out
if (tableId == null || tableId.isEmpty()) {
tableId = getParamValue(url,"docid");
}
return tableId;
}
static public Fusiontables getFusionTablesService(String token) {
Credential credential = new GoogleCredential().setAccessToken(token);
Fusiontables fusiontables = new Fusiontables.Builder(
GDataExtension.HTTP_TRANSPORT, GDataExtension.JSON_FACTORY, credential)
.setApplicationName(GDataExtension.SERVICE_APP_NAME)
.build();;
return fusiontables;
}
static boolean isFusionTableURL(URL url) {
// http://www.google.com/fusiontables/DataSource?dsrcid=1219
String query = url.getQuery();
if (query == null) {
query = "";
}
return url.getHost().endsWith(".google.com")
&& url.getPath().startsWith("/fusiontables/DataSource")
&& (query.contains("dsrcid=")||query.contains("docid="));
}
static Sqlresponse runFusionTablesSelect(Fusiontables service, String selectQuery)
throws IOException {
// FIXME: alt=csv doesn't actually work! It will attempt to parse response as JSON and die
// perhaps use .executeUnparsed() would work?
SqlGet query = service.query().sqlGet(selectQuery);//.setAlt("csv");
Sqlresponse response = query.execute();
return response;
}
static private String getParamValue(URL url, String key) {
String query = url.getQuery();
if (query != null) {
String[] parts = query.split("&");
for (String part : parts) {
if (part.startsWith(key+"=")) {
int offset = key.length()+1;
String tableId = part.substring(offset);
return tableId;
}
}
}
return null;
}
}

284
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java Normal file
View File

@ -0,0 +1,284 @@
/*
* Copyright (c) 2010,2013 Thomas F. Morris and other contributors
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONObject;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.model.Column;
import com.google.api.services.fusiontables.model.Sqlresponse;
import com.google.api.services.fusiontables.model.Table;
import com.google.refine.ProjectMetadata;
import com.google.refine.importers.TabularImportingParserBase;
import com.google.refine.importers.TabularImportingParserBase.TableDataReader;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.util.JSONUtilities;
/**
* OpenRefine parser for Google Spreadsheets.
*
* @author Tom Morris <tfmorris@gmail.com>
* @copyright 2010 Thomas F. Morris
* @license New BSD http://www.opensource.org/licenses/bsd-license.php
*/
public class FusionTableImporter {
static public void parse(
String token,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
Fusiontables service = FusionTableHandler.getFusionTablesService(token);
parse(
service,
project,
metadata,
job,
limit,
options,
exceptions
);
}
static void setProgress(ImportingJob job, String fileSource, int percent) {
job.setProgress(percent, "Reading " + fileSource);
}
static private class FusionTableBatchRowReader implements TableDataReader {
final ImportingJob job;
final String fileSource;
final Fusiontables service;
final List<FTColumnData> columns;
final int batchSize;
final String baseQuery;
int nextRow = 0; // 0-based
int batchRowStart = 0; // 0-based
boolean end = false;
List<List<Object>> rowsOfCells = null;
boolean usedHeaders = false;
public FusionTableBatchRowReader(ImportingJob job, String fileSource,
Fusiontables service, String tableId, List<FTColumnData> columns,
int batchSize) {
this.job = job;
this.fileSource = fileSource;
this.service = service;
this.columns = columns;
this.batchSize = batchSize;
StringBuffer sb = new StringBuffer();
sb.append("SELECT ");
boolean first = true;
for (FTColumnData cd : columns) {
if (first) {
first = false;
} else {
sb.append(",");
}
sb.append("'");
sb.append(cd.name);
sb.append("'");
}
sb.append(" FROM ");
sb.append(tableId);
baseQuery = sb.toString();
}
@Override
public List<Object> getNextRowOfCells() throws IOException {
if (!usedHeaders) {
List<Object> row = new ArrayList<Object>(columns.size());
for (FTColumnData cd : columns) {
row.add(cd.name);
}
usedHeaders = true;
return row;
}
if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) {
int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
rowsOfCells = getRowsOfCells(newBatchRowStart);
batchRowStart = newBatchRowStart;
setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */);
}
if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
return rowsOfCells.get(nextRow++ - batchRowStart);
} else {
return null;
}
}
private List<List<Object>> getRowsOfCells(int startRow) throws IOException {
List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(batchSize);
String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize;
Sqlresponse sqlresponse = FusionTableHandler.runFusionTablesSelect(service, query);
List<List<Object>> rows = sqlresponse.getRows();
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<Object> row = rows.get(i);
List<Object> rowOfCells = new ArrayList<Object>(row.size());
for (int j = 0; j < row.size() && j < columns.size(); j++) {
String text = (String)row.get(j);
if (text.isEmpty()) {
rowOfCells.add(null);
} else {
FTColumnData cd = columns.get(j);
if (cd.type == FTColumnType.NUMBER) {
try {
rowOfCells.add(Long.parseLong(text));
continue;
} catch (NumberFormatException e) {
// ignore
}
try {
double d = Double.parseDouble(text);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
rowOfCells.add(d);
continue;
}
} catch (NumberFormatException e) {
// ignore
}
}
rowOfCells.add(text);
}
}
rowsOfCells.add(rowOfCells);
}
}
end = rows.size() < batchSize + 1;
return rowsOfCells;
}
}
static public void parse(
Fusiontables service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
// TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?
try {
List<FTColumnData> columns = new ArrayList<FusionTableImporter.FTColumnData>();
Table table = service.table().get(id).execute();
for (Column col : table.getColumns()) {
FTColumnData cd = new FTColumnData();
cd.name = col.getName();
String type = col.getType();
if (type.equals("STRING")) {
cd.type = FTColumnType.STRING;
} else if (type.equals("NUMBER")) {
cd.type = FTColumnType.NUMBER;
} else if (type.equals("DATETIME")) {
cd.type = FTColumnType.DATETIME;
} else if (type.equals("LOCATION")) {
cd.type = FTColumnType.LOCATION;
} else {
// TODO: unknown type
cd.type = FTColumnType.STRING;
}
columns.add(cd);
}
setProgress(job, docUrlString, -1);
// Force these options for the next call because each fusion table
// is strictly structured with a single line of headers.
JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
TabularImportingParserBase.readTable(
project,
metadata,
job,
new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
docUrlString,
limit,
options,
exceptions
);
setProgress(job, docUrlString, 100);
} catch (IOException e) {
e.printStackTrace();
exceptions.add(e);
}
}
static private String getFTid(String url) {
if (url == null) {
return null;
}
int equal = url.lastIndexOf('=');
if (equal < 0) {
return null;
}
return url.substring(equal + 1);
}
static enum FTColumnType {
STRING,
NUMBER,
DATETIME,
LOCATION
}
final static class FTColumnData {
String name;
FTColumnType type;
}
}

156
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java Normal file
View File

@ -0,0 +1,156 @@
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONObject;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpResponseException;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.refine.exporters.TabularSerializer;
final class FusionTableSerializer implements TabularSerializer {
private static final int BATCH_SIZE = 20;
Fusiontables service;
String tableName;
List<Exception> exceptions;
String tableId;
List<String> columnNames;
StringBuffer sbBatch;
int rows;
FusionTableSerializer(Fusiontables service, String tableName, List<Exception> exceptions) {
this.service = service;
this.tableName = tableName;
this.exceptions = exceptions;
}
@Override
public void startFile(JSONObject options) {
}
@Override
public void endFile() {
if (sbBatch != null) {
sendBatch();
}
}
@Override
public void addRow(List<CellData> cells, boolean isHeader) {
if (isHeader) {
columnNames = new ArrayList<String>(cells.size());
for (CellData cellData : cells) {
columnNames.add(cellData.text);
}
try {
tableId = FusionTableHandler.createTable(service, tableName, columnNames);
} catch (Exception e) {
tableId = null;
exceptions.add(e);
}
} else if (tableId != null) {
if (sbBatch == null) {
sbBatch = new StringBuffer();
}
formatCsv(cells, sbBatch);
rows++;
if (rows % BATCH_SIZE == 0) {
if (!sendBatch()) {
return;
}
}
}
}
private boolean sendBatch() {
try {
// FIXME: text/csv doesn't work even though that's what the content is
AbstractInputStreamContent content = ByteArrayContent.fromString("application/octet-stream", sbBatch.toString());
// AbstractInputStreamContent content = new InputStreamContent("application/octet-stream",
// // TODO: we really want to do GZIP compression here
// new ByteArrayInputStream(sbBatch.toString().getBytes("UTF-8")));
Long count = FusionTableHandler.insertRows(service, tableId, content);
if (count != BATCH_SIZE) {
exceptions.add(new IOException("only imported %d of %d rows"));
}
} catch (IOException e) {
exceptions.add(e);
if (e instanceof HttpResponseException) {
int code = ((HttpResponseException)e).getStatusCode();
if (code >= 400 && code < 500) {
return false;
}
// 500s appear to be retried automatically by li
}
} finally {
sbBatch = null;
}
return true;
}
private void formatCsv(List<CellData> cells, StringBuffer sb) {
boolean first = true;
for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
CellData cellData = cells.get(i);
if (!first) {
sb.append(',');
} else {
first = false;
}
sb.append("\"");
if (cellData != null && cellData.text != null) {
sb.append(cellData.text.replaceAll("\"", "\\\\\""));
}
sb.append("\"");
}
sb.append("\n");
}
// Old-style SQL INSERT can be removed once we're sure importRows will work
private void formulateInsert(List<CellData> cells, StringBuffer sb) {
StringBuffer sbColumnNames = new StringBuffer();
StringBuffer sbValues = new StringBuffer();
boolean first = true;
for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
CellData cellData = cells.get(i);
if (first) {
first = false;
} else {
sbColumnNames.append(',');
sbValues.append(',');
}
sbColumnNames.append("'");
sbColumnNames.append(columnNames.get(i));
sbColumnNames.append("'");
sbValues.append("'");
if (cellData != null && cellData.text != null) {
sbValues.append(cellData.text.replaceAll("'", "\\\\'"));
}
sbValues.append("'");
}
if (sb.length() > 0) {
sb.append(';');
}
sb.append("INSERT INTO ");
sb.append(tableId);
sb.append("(");
sb.append(sbColumnNames.toString());
sb.append(") values (");
sb.append(sbValues.toString());
sb.append(")");
}
public String getUrl() {
return tableId == null || exceptions.size() > 0 ? null :
"https://www.google.com/fusiontables/DataSource?docid=" + tableId;
}
}

196
extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java
View File

@ -29,27 +29,22 @@
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.Arrays;
import javax.servlet.http.HttpServletRequest;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.api.client.auth.oauth2.AuthorizationCodeResponseUrl;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest;
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.http.AuthSubUtil;
import com.google.gdata.client.spreadsheet.FeedURLFactory;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.util.ContentType;
import com.google.gdata.util.ServiceException;
import com.google.refine.util.ParsingUtilities;
@ -62,37 +57,73 @@ import edu.mit.simile.butterfly.ButterflyModule;
*/
abstract public class GDataExtension {
static final String SERVICE_APP_NAME = "OpenRefine-GData-Extension";
static final String CLIENT_ID = "647865400439.apps.googleusercontent.com";
static final String CLIENT_SECRET = "0mW9OJji1yrgJk5AjJc5Pn6I"; // not really that secret, but the protocol accounts for that
/** Global instance of the HTTP transport. */
static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
/** Global instance of the JSON factory. */
static final JsonFactory JSON_FACTORY = new JacksonFactory();
static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request)
throws MalformedURLException {
char[] mountPointChars = module.getMountPoint().getMountPoint().toCharArray();
StringBuffer sb = new StringBuffer();
sb.append(mountPointChars, 0, mountPointChars.length);
String authorizedUrl = makeRedirectUrl(module, request);
// New Oauth2
GoogleAuthorizationCodeRequestUrl url = new GoogleAuthorizationCodeRequestUrl(
CLIENT_ID,
authorizedUrl, // execution continues at authorized on redirect
Arrays.asList("https://www.googleapis.com/auth/fusiontables",
"https://docs.google.com/feeds", // create new spreadsheets
"https://spreadsheets.google.com/feeds"));
return url.toString();
}
private static String makeRedirectUrl(ButterflyModule module, HttpServletRequest request)
throws MalformedURLException {
StringBuffer sb = new StringBuffer(module.getMountPoint().getMountPoint());
sb.append("authorized?winname=");
sb.append(ParsingUtilities.encode(request.getParameter("winname")));
sb.append("&callback=");
sb.append(ParsingUtilities.encode(request.getParameter("callback")));
sb.append("&cb=");
sb.append(ParsingUtilities.encode(request.getParameter("cb")));
URL thisUrl = new URL(request.getRequestURL().toString());
URL authorizedUrl = new URL(thisUrl, sb.toString());
return AuthSubUtil.getRequestUrl(
authorizedUrl.toExternalForm(), // execution continues at authorized on redirect
"https://docs.google.com/feeds https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query",
false,
true);
return authorizedUrl.toExternalForm();
}
static private FeedURLFactory factory;
static public FeedURLFactory getFeedUrlFactory() {
if (factory == null) {
// Careful - this is shared by everyone.
factory = FeedURLFactory.getDefault();
static public String getTokenFromCode(ButterflyModule module, HttpServletRequest request)
throws MalformedURLException {
String redirectUrl = makeRedirectUrl(module, request);
StringBuffer fullUrlBuf = request.getRequestURL();
if (request.getQueryString() != null) {
fullUrlBuf.append('?').append(request.getQueryString());
}
return factory;
}
AuthorizationCodeResponseUrl authResponse =
new AuthorizationCodeResponseUrl(fullUrlBuf.toString());
// check for user-denied error
if (authResponse.getError() != null) {
// authorization denied...
} else {
// request access token using authResponse.getCode()...
String code = authResponse.getCode();
try {
GoogleTokenResponse response = new GoogleAuthorizationCodeTokenRequest(HTTP_TRANSPORT,
JSON_FACTORY, CLIENT_ID, CLIENT_SECRET, code, redirectUrl).execute();
String token = response.getAccessToken();
return token;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return null;
}
static public DocsService getDocsService(String token) {
DocsService service = new DocsService(SERVICE_APP_NAME);
if (token != null) {
@ -109,83 +140,6 @@ abstract public class GDataExtension {
return service;
}
static public GoogleService getFusionTablesGoogleService(String token) {
GoogleService service = new GoogleService("fusiontables", SERVICE_APP_NAME);
if (token != null) {
service.setAuthSubToken(token);
}
return service;
}
final static private String FUSION_TABLES_SERVICE_URL =
"https://www.google.com/fusiontables/api/query";
final static private Pattern CSV_VALUE_PATTERN =
Pattern.compile("([^,\\r\\n\"]*|\"(([^\"]*\"\")*[^\"]*)\")(,|\\r?\\n)");
static public List<List<String>> runFusionTablesSelect(GoogleService service, String selectQuery)
throws IOException, ServiceException {
GDataRequest request = createFusionTablesRequest(service, RequestType.QUERY, selectQuery);
request.execute();
return parseFusionTablesResults(request);
}
static public GDataRequest createFusionTablesRequest(
GoogleService service, RequestType requestType, String query)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
URLEncoder.encode(query, "UTF-8"));
return service.getRequestFactory().getRequest(
requestType, url, ContentType.TEXT_PLAIN);
}
static public GDataRequest createFusionTablesPostRequest(
GoogleService service, RequestType requestType, String query)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL);
GDataRequest request = service.getRequestFactory().getRequest(
requestType, url, new ContentType("application/x-www-form-urlencoded"));
OutputStreamWriter writer =
new OutputStreamWriter(request.getRequestStream());
writer.append("sql=" + URLEncoder.encode(query, "UTF-8"));
writer.flush();
writer.close();
return request;
}
static public List<List<String>> parseFusionTablesResults(GDataRequest request) throws IOException {
List<List<String>> rows = new ArrayList<List<String>>();
List<String> row = null;
Scanner scanner = new Scanner(request.getResponseStream(), "UTF-8");
while (scanner.hasNextLine()) {
scanner.findWithinHorizon(CSV_VALUE_PATTERN, 0);
MatchResult match = scanner.match();
String quotedString = match.group(2);
String decoded = quotedString == null ? match.group(1) : quotedString.replaceAll("\"\"", "\"");
if (row == null) {
row = new ArrayList<String>();
}
row.add(decoded);
if (!match.group(4).equals(",")) {
if (row != null) {
rows.add(row);
row = null;
}
}
}
scanner.close();
if (row != null) {
rows.add(row);
}
return rows;
}
static boolean isSpreadsheetURL(String url) {
// e.g. http://spreadsheets.google.com/ccc?key=tI36b9Fxk1lFBS83iR_3XQA&hl=en
// TODO: The following should work, but the GData implementation is too limited
@ -220,25 +174,5 @@ abstract public class GDataExtension {
}
return null;
}
static boolean isFusionTableURL(URL url) {
// http://www.google.com/fusiontables/DataSource?dsrcid=1219
String query = url.getQuery();
if (query == null) {
query = "";
}
return url.getHost().endsWith(".google.com")
&& url.getPath().startsWith("/fusiontables/DataSource")
&& query.contains("dsrcid=");
}
static String getFusionTableKey(URL url) {
String tableId = getParamValue(url,"dsrcid");
// TODO: Any special id format considerations to worry about?
// if (tableId.startsWith("p") || !tableId.contains(".")) {
// return tableId;
// }
return tableId;
}
}

224
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -36,7 +36,6 @@ import java.util.List;
import org.json.JSONObject;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.spreadsheet.CellQuery;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.Cell;
@ -83,9 +82,7 @@ public class GDataImporter {
exceptions
);
} else if ("table".equals(docType)) {
GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
parse(
service,
FusionTableImporter.parse(token,
project,
metadata,
job,
@ -172,13 +169,7 @@ public class GDataImporter {
}
static private void setProgress(ImportingJob job, String fileSource, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
JSONUtilities.safePut(progress, "percent", percent);
job.setProgress(percent, "Reading " + fileSource);
}
static private class WorksheetBatchRowReader implements TableDataReader {
@ -278,214 +269,5 @@ public class GDataImporter {
return rowsOfCells;
}
}
}
static public void parse(
GoogleService service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
// TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?
try {
List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id);
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
if (row.size() >= 2) {
FTColumnData cd = new FTColumnData();
cd.name = row.get(1);
cd.type = FTColumnType.STRING;
if (row.size() > 2) {
String type = row.get(2).toLowerCase();
if (type.equals("number")) {
cd.type = FTColumnType.NUMBER;
} else if (type.equals("datetime")) {
cd.type = FTColumnType.DATETIME;
} else if (type.equals("location")) {
cd.type = FTColumnType.LOCATION;
}
}
columns.add(cd);
}
}
setProgress(job, docUrlString, -1);
// Force these options for the next call because each fusion table
// is strictly structured with a single line of headers.
JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
TabularImportingParserBase.readTable(
project,
metadata,
job,
new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
docUrlString,
limit,
options,
exceptions
);
setProgress(job, docUrlString, 100);
}
} catch (IOException e) {
e.printStackTrace();
exceptions.add(e);
} catch (ServiceException e) {
e.printStackTrace();
exceptions.add(e);
}
}
static private String getFTid(String url) {
if (url == null) {
return null;
}
int equal = url.lastIndexOf('=');
if (equal < 0) {
return null;
}
return url.substring(equal + 1);
}
static private enum FTColumnType {
STRING,
NUMBER,
DATETIME,
LOCATION
}
final static private class FTColumnData {
String name;
FTColumnType type;
}
static private class FusionTableBatchRowReader implements TableDataReader {
final ImportingJob job;
final String fileSource;
final GoogleService service;
final List<FTColumnData> columns;
final int batchSize;
final String baseQuery;
int nextRow = 0; // 0-based
int batchRowStart = 0; // 0-based
boolean end = false;
List<List<Object>> rowsOfCells = null;
boolean usedHeaders = false;
public FusionTableBatchRowReader(ImportingJob job, String fileSource,
GoogleService service, String tableId, List<FTColumnData> columns,
int batchSize) {
this.job = job;
this.fileSource = fileSource;
this.service = service;
this.columns = columns;
this.batchSize = batchSize;
StringBuffer sb = new StringBuffer();
sb.append("SELECT ");
boolean first = true;
for (FTColumnData cd : columns) {
if (first) {
first = false;
} else {
sb.append(",");
}
sb.append("'");
sb.append(cd.name);
sb.append("'");
}
sb.append(" FROM ");
sb.append(tableId);
baseQuery = sb.toString();
}
@Override
public List<Object> getNextRowOfCells() throws IOException {
if (!usedHeaders) {
List<Object> row = new ArrayList<Object>(columns.size());
for (FTColumnData cd : columns) {
row.add(cd.name);
}
usedHeaders = true;
return row;
}
if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) {
int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
try {
rowsOfCells = getRowsOfCells(newBatchRowStart);
batchRowStart = newBatchRowStart;
setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */);
} catch (ServiceException e) {
throw new IOException(e);
}
}
if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
return rowsOfCells.get(nextRow++ - batchRowStart);
} else {
return null;
}
}
private List<List<Object>> getRowsOfCells(int startRow) throws IOException, ServiceException {
List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(batchSize);
String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize;
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, query);
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
List<Object> rowOfCells = new ArrayList<Object>(row.size());
for (int j = 0; j < row.size() && j < columns.size(); j++) {
String text = row.get(j);
if (text.isEmpty()) {
rowOfCells.add(null);
} else {
FTColumnData cd = columns.get(j);
if (cd.type == FTColumnType.NUMBER) {
try {
rowOfCells.add(Long.parseLong(text));
continue;
} catch (NumberFormatException e) {
// ignore
}
try {
double d = Double.parseDouble(text);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
rowOfCells.add(d);
continue;
}
} catch (NumberFormatException e) {
// ignore
}
}
rowOfCells.add(text);
}
}
rowsOfCells.add(rowOfCells);
}
}
end = rows.size() < batchSize + 1;
return rowsOfCells;
}
}
}

63
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -49,7 +49,9 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.gdata.client.GoogleService;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.model.Table;
import com.google.api.services.fusiontables.model.TableList;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.spreadsheet.FeedURLFactory;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
@ -59,6 +61,7 @@ import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
import com.google.gdata.data.spreadsheet.WorksheetEntry;
import com.google.gdata.data.spreadsheet.WorksheetFeed;
import com.google.gdata.util.AuthenticationException;
import com.google.gdata.util.ServiceException;
import com.google.refine.ProjectManager;
@ -125,13 +128,15 @@ public class GDataImportingController implements ImportingController {
try {
listSpreadsheets(GDataExtension.getDocsService(token), writer);
listFusionTables(GDataExtension.getFusionTablesGoogleService(token), writer);
listFusionTables(FusionTableHandler.getFusionTablesService(token), writer);
} catch (AuthenticationException e) {
TokenCookie.deleteToken(request, response);
} catch (ServiceException e) {
e.printStackTrace();
} finally {
writer.endArray();
writer.endObject();
}
writer.endArray();
writer.endObject();
} catch (JSONException e) {
throw new ServletException(e);
} finally {
@ -167,27 +172,24 @@ public class GDataImportingController implements ImportingController {
}
}
private void listFusionTables(GoogleService service, JSONWriter writer)
private void listFusionTables(Fusiontables service, JSONWriter writer)
throws IOException, ServiceException, JSONException {
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "SHOW TABLES");
if (rows.size() > 1) { // excluding headers
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
if (row.size() >= 2) {
String id = row.get(0);
String name = row.get(1);
String link = "https://www.google.com/fusiontables/DataSource?dsrcid=" + id;
writer.object();
writer.key("docId"); writer.value(id);
writer.key("docLink"); writer.value(link);
writer.key("docSelfLink"); writer.value(link);
writer.key("title"); writer.value(name);
writer.key("type"); writer.value("table");
writer.endObject();
}
}
Fusiontables.Table.List listTables = service.table().list();
TableList tablelist = listTables.execute();
for (Table table : tablelist.getItems()) {
String id = table.getTableId();
String name = table.getName();
String link = "https://www.google.com/fusiontables/DataSource?docid=" + id;
// Add JSON object to our stream
writer.object();
writer.key("docId"); writer.value(id);
writer.key("docLink"); writer.value(link);
writer.key("docSelfLink"); writer.value(link);
writer.key("title"); writer.value(name);
writer.key("type"); writer.value("table");
writer.endObject();
}
}
@ -300,9 +302,6 @@ public class GDataImportingController implements ImportingController {
job.updating = true;
try {
// This is for setting progress during the parsing process.
job.config = new JSONObject();
JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
request.getParameter("options"));
@ -371,7 +370,7 @@ public class GDataImportingController implements ImportingController {
final List<Exception> exceptions = new LinkedList<Exception>();
JSONUtilities.safePut(job.config, "state", "creating-project");
job.setState("creating-project");
final Project project = new Project();
new Thread() {
@ -393,16 +392,14 @@ public class GDataImportingController implements ImportingController {
if (!job.canceled) {
if (exceptions.size() > 0) {
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
JSONUtilities.safePut(job.config, "state", "error");
job.setError(exceptions);
} else {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "state", "created-project");
JSONUtilities.safePut(job.config, "projectID", project.id);
job.setState("created-project");
job.setProjectID(project.id);
}
job.touch();

2
extensions/gdata/src/com/google/refine/extension/gdata/TokenCookie.java
View File

@ -8,7 +8,7 @@ import com.google.refine.util.CookiesUtilities;
public class TokenCookie {
private static final String COOKIE_NAME = "authsub_token";
private static final String COOKIE_NAME = "oauth2_token";
private static int MAX_AGE = 30 * 24 * 60 * 60; // 30 days
public static String getToken(HttpServletRequest request) {

141
extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java
View File

@ -33,7 +33,6 @@ import java.io.IOException;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
@ -47,9 +46,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.spreadsheet.CellQuery;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
@ -295,142 +291,13 @@ public class UploadCommand extends Command {
static private String uploadFusionTable(
Project project, final Engine engine, final Properties params,
String token, String name, List<Exception> exceptions) {
GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
FusionTableSerializer serializer = new FusionTableSerializer(service, name, exceptions);
FusionTableSerializer serializer = new FusionTableSerializer(
FusionTableHandler.getFusionTablesService(token), name, exceptions);
CustomizableTabularExporterUtilities.exportRows(
project, engine, params, serializer);
return serializer.tableId == null || exceptions.size() > 0 ? null :
"https://www.google.com/fusiontables/DataSource?dsrcid=" + serializer.tableId;
}
final static private class FusionTableSerializer implements TabularSerializer {
GoogleService service;
String tableName;
List<Exception> exceptions;
String tableId;
List<String> columnNames;
StringBuffer sbBatch;
int rows;
FusionTableSerializer(GoogleService service, String tableName, List<Exception> exceptions) {
this.service = service;
this.tableName = tableName;
this.exceptions = exceptions;
}
@Override
public void startFile(JSONObject options) {
}
@Override
public void endFile() {
if (sbBatch != null) {
sendBatch();
}
}
@Override
public void addRow(List<CellData> cells, boolean isHeader) {
if (isHeader) {
columnNames = new ArrayList<String>(cells.size());
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE '");
sb.append(tableName);
sb.append("' (");
boolean first = true;
for (CellData cellData : cells) {
columnNames.add(cellData.text);
if (first) {
first = false;
} else {
sb.append(',');
}
sb.append("'");
sb.append(cellData.text);
sb.append("': STRING");
}
sb.append(")");
try {
String createQuery = sb.toString();
GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
service, RequestType.INSERT, createQuery);
createTableRequest.execute();
List<List<String>> createTableResults =
GDataExtension.parseFusionTablesResults(createTableRequest);
if (createTableResults != null && createTableResults.size() == 2) {
tableId = createTableResults.get(1).get(0);
}
} catch (Exception e) {
exceptions.add(e);
}
} else if (tableId != null) {
if (sbBatch == null) {
sbBatch = new StringBuffer();
}
formulateInsert(cells, sbBatch);
rows++;
if (rows % 20 == 0) {
sendBatch();
}
}
}
void sendBatch() {
try {
GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
service, RequestType.INSERT, sbBatch.toString());
createTableRequest.execute();
} catch (IOException e) {
exceptions.add(e);
} catch (ServiceException e) {
exceptions.add(e);
} finally {
sbBatch = null;
}
}
void formulateInsert(List<CellData> cells, StringBuffer sb) {
StringBuffer sbColumnNames = new StringBuffer();
StringBuffer sbValues = new StringBuffer();
boolean first = true;
for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
CellData cellData = cells.get(i);
if (first) {
first = false;
} else {
sbColumnNames.append(',');
sbValues.append(',');
}
sbColumnNames.append("'");
sbColumnNames.append(columnNames.get(i));
sbColumnNames.append("'");
sbValues.append("'");
if (cellData != null && cellData.text != null) {
sbValues.append(cellData.text.replaceAll("'", "\\\\'"));
}
sbValues.append("'");
}
if (sb.length() > 0) {
sb.append(';');
}
sb.append("INSERT INTO ");
sb.append(tableId);
sb.append("(");
sb.append(sbColumnNames.toString());
sb.append(") values (");
sb.append(sbValues.toString());
sb.append(")");
}
return serializer.getUrl();
}
}

74
extensions/gdata/src/com/google/refine/extension/gdata/commands/LoadLanguageCommand.java
View File

@ -1,74 +0,0 @@
package com.google.refine.extension.gdata.commands;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
public class LoadLanguageCommand extends Command {
public LoadLanguageCommand() {
// TODO Auto-generated constructor stub
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String rawDirectoryFile = request.getSession().getServletContext()
.getRealPath("extensions/gdata/module/langs/");
String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
BufferedReader reader = null;
String param = null;
try {
param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
} catch (NullPointerException e) {
}
if (param == null) param = request.getParameter("lng");
String[] langs = param.split(" ");
try {
String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e1) {
try {
String file = cleanedDirectory + File.separator + "translation-default.json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e3) {
e3.printStackTrace();
}
}
String line = null;
String message = new String();
if (reader != null) {
while ((line = reader.readLine()) != null) {
// buffer.append(line);
message += line + System.getProperty("line.separator");
}
}
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.getWriter().println(message);
response.getWriter().flush();
response.getWriter().close();
}
}

6
extensions/pc-axis/module/scripts/pc-axis-parser-ui.js
View File

@ -110,15 +110,15 @@ Refine.PCAxisParserUI.prototype._initialize = function() {
});
if (this._config.limit > 0) {
this._optionContainerElmts.limitCheckbox.attr("checked", "checked");
this._optionContainerElmts.limitCheckbox.prop("checked", true);
this._optionContainerElmts.limitInput[0].value = this._config.limit.toString();
}
if (this._config.skipDataLines > 0) {
this._optionContainerElmts.skipCheckbox.attr("checked", "checked");
this._optionContainerElmts.skipCheckbox.prop("checked", true);
this._optionContainerElmts.skipInput.value[0].value = this._config.skipDataLines.toString();
}
if (this._config.includeFileSources) {
this._optionContainerElmts.includeFileSourcesCheckbox.attr("checked", "checked");
this._optionContainerElmts.includeFileSourcesCheckbox.prop("checked", true);
}
var onChange = function() {

11
main/src/com/google/refine/InterProjectModel.java
View File

@ -35,6 +35,7 @@ package com.google.refine;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -109,10 +110,11 @@ public class InterProjectModel {
public void flushJoinsInvolvingProject(long projectID) {
synchronized (_joins) {
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
for (Iterator<Entry<String, ProjectJoin>> it = _joins.entrySet().iterator(); it.hasNext();) {
Entry<String, ProjectJoin> entry = it.next();
ProjectJoin join = entry.getValue();
if (join.fromProjectID == projectID || join.toProjectID == projectID) {
_joins.remove(entry.getKey());
it.remove();
}
}
}
@ -120,11 +122,12 @@ public class InterProjectModel {
public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) {
synchronized (_joins) {
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
for (Iterator<Entry<String, ProjectJoin>> it = _joins.entrySet().iterator(); it.hasNext();) {
Entry<String, ProjectJoin> entry = it.next();
ProjectJoin join = entry.getValue();
if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) ||
join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) {
_joins.remove(entry.getKey());
it.remove();
}
}
}

41
main/src/com/google/refine/ProjectManager.java
View File

@ -62,6 +62,13 @@ public abstract class ProjectManager {
// last n expressions used across all projects
static protected final int s_expressionHistoryMax = 100;
// If a project has been idle this long, flush it from memory
static protected final int PROJECT_FLUSH_DELAY = 1000 * 60 * 15; // 15 minutes
// Don't spend more than this much time saving projects if doing a quick save
static protected final int QUICK_SAVE_MAX_TIME = 1000 * 30; // 30 secs
protected Map<Long, ProjectMetadata> _projectsMetadata;
protected PreferenceStore _preferenceStore;
@ -119,7 +126,6 @@ public abstract class ProjectManager {
_projectsMetadata.put(project.id, projectMetadata);
}
}
//----------Load from data store to memory----------------
/**
* Load project metadata from data storage
@ -135,7 +141,6 @@ public abstract class ProjectManager {
*/
protected abstract Project loadProject(long id);
//------------Import and Export from Refine archive-----------------
/**
* Import project from a Refine archive
* @param projectID
@ -154,7 +159,6 @@ public abstract class ProjectManager {
public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException;
//------------Save to record store------------
/**
* Saves a project and its metadata to the data store
* @param id
@ -194,8 +198,9 @@ public abstract class ProjectManager {
/**
* Save project to the data store
* @param project
* @throws IOException
*/
protected abstract void saveProject(Project project);
protected abstract void saveProject(Project project) throws IOException;
/**
* Save workspace and all projects to data store
@ -227,9 +232,6 @@ public abstract class ProjectManager {
}
}
static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
/**
* Saves all projects to the data store
* @param allModified
@ -256,7 +258,7 @@ public abstract class ProjectManager {
records.add(new SaveRecord(project, msecsOverdue));
} else if (!project.getProcessManager().hasPending()
&& startTimeOfSave.getTime() - project.getLastSave().getTime() > s_projectFlushDelay) {
&& startTimeOfSave.getTime() - project.getLastSave().getTime() > PROJECT_FLUSH_DELAY) {
/*
* It's been a while since the project was last saved and it hasn't been
@ -289,19 +291,36 @@ public abstract class ProjectManager {
for (int i = 0;
i < records.size() &&
(allModified || (new Date().getTime() - startTimeOfSave.getTime() < s_quickSaveTimeout));
(allModified || (new Date().getTime() - startTimeOfSave.getTime() < QUICK_SAVE_MAX_TIME));
i++) {
try {
saveProject(records.get(i).project);
} catch (Exception e) {
e.printStackTrace();
// In case we're running low on memory, free as much as we can
disposeUnmodifiedProjects();
}
}
}
}
/**
* Flush all unmodified projects from memory.
*/
protected void disposeUnmodifiedProjects() {
synchronized (this) {
for (long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = getProjectMetadata(id);
Project project = _projects.get(id);
if (project != null && !project.getProcessManager().hasPending()
&& metadata.getModified().getTime() < project.getLastSave().getTime()) {
_projects.remove(id).dispose();
}
}
}
}
//--------------Get from memory--------------
/**
* Gets the InterProjectModel from memory
*/
@ -405,7 +424,6 @@ public abstract class ProjectManager {
*/
public abstract HistoryEntryManager getHistoryEntryManager();
//-------------remove project-----------
/**
* Remove the project from the data store
@ -434,7 +452,6 @@ public abstract class ProjectManager {
}
}
//--------------Miscellaneous-----------
/**
* Sets the flag for long running operations. This will prevent

40
main/src/com/google/refine/ProjectMetadata.java
View File

@ -54,6 +54,7 @@ import com.google.refine.util.ParsingUtilities;
public class ProjectMetadata implements Jsonizable {
private final Date _created;
private Date _modified;
private Date written = null;
private String _name;
private String _password;
@ -71,11 +72,16 @@ public class ProjectMetadata implements Jsonizable {
}
public ProjectMetadata() {
_created = new Date();
this(new Date());
_modified = _created;
preparePreferenceStore(_preferenceStore);
}
public ProjectMetadata(Date created, Date modified, String name) {
this(created);
_modified = modified;
_name = name;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
@ -103,16 +109,36 @@ public class ProjectMetadata implements Jsonizable {
}
writer.endObject();
if ("save".equals(options.getProperty("mode"))) {
written = new Date();
}
}
public boolean isDirty() {
return written == null || _modified.after(written);
}
public void write(JSONWriter jsonWriter) throws Exception {
Properties options = new Properties();
options.setProperty("mode", "save");
public void write(JSONWriter jsonWriter) throws JSONException {
write(jsonWriter, false);
}
/**
* @param jsonWriter writer to save metadatea to
* @param onlyIfDirty true to not write unchanged metadata
* @throws JSONException
*/
public void write(JSONWriter jsonWriter, boolean onlyIfDirty) throws JSONException {
if (!onlyIfDirty || isDirty()) {
Properties options = new Properties();
options.setProperty("mode", "save");
write(jsonWriter, options);
write(jsonWriter, options);
}
}
static public ProjectMetadata loadFromJSON(JSONObject obj) {
// TODO: Is this correct? It's using modified date for creation date
ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date()));
pm._modified = JSONUtilities.getDate(obj, "modified", new Date());
@ -156,6 +182,8 @@ public class ProjectMetadata implements Jsonizable {
// ignore
}
}
pm.written = new Date(); // Mark it as not needing writing until modified
return pm;
}

27
main/src/com/google/refine/RefineServlet.java
View File

@ -41,8 +41,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -80,23 +81,19 @@ public class RefineServlet extends Butterfly {
static final private Map<String, Command> commands = new HashMap<String, Command>();
// timer for periodically saving projects
static private Timer _timer = new Timer("autosave");
static private ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
static final Logger logger = LoggerFactory.getLogger("refine");
static final protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
static final protected long AUTOSAVE_PERIOD = 5; // 5 minutes
static protected class AutoSaveTimerTask extends TimerTask {
static protected class AutoSaveTimerTask implements Runnable {
@Override
public void run() {
try {
ProjectManager.singleton.save(false); // quick, potentially incomplete save
} finally {
if (_timer != null) {
_timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
// we don't use scheduleAtFixedRate because that might result in
// bunched up events when the computer is put in sleep mode
}
} catch (final Throwable e) {
// Not the best, but we REALLY want this to keep trying
}
}
}
@ -134,7 +131,8 @@ public class RefineServlet extends Butterfly {
FileProjectManager.initialize(s_dataDir);
ImportingManager.initialize(this);
_timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
service.scheduleWithFixedDelay(new AutoSaveTimerTask(), AUTOSAVE_PERIOD,
AUTOSAVE_PERIOD, TimeUnit.MINUTES);
logger.trace("< initialize");
}
@ -370,7 +368,10 @@ public class RefineServlet extends Butterfly {
}
static public void setUserAgent(HttpURLConnection httpConnection) {
httpConnection.addRequestProperty("User-Agent", "OpenRefine/" + FULL_VERSION);
httpConnection.addRequestProperty("User-Agent", getUserAgent());
}
static public String getUserAgent() {
return "OpenRefine/" + FULL_VERSION;
}
}

114
main/src/com/google/refine/commands/lang/GetLanguagesCommand.java Normal file
View File

@ -0,0 +1,114 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.commands.lang;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import edu.mit.simile.butterfly.ButterflyModule;
public class GetLanguagesCommand extends Command {
public GetLanguagesCommand() {
super();
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String modname = request.getParameter("module");
if (modname == null) {
modname = "core";
}
ButterflyModule module = this.servlet.getModule(modname);
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("languages");
writer.array();
writeLangData(writer, "en", "English"); // we always have English and it's always first
FileFilter fileFilter = new WildcardFileFilter("translation-*.json");
for (File file : new File(module.getPath() + File.separator + "langs").listFiles(fileFilter)) {
String lang = file.getName().split("-")[1].split("\\.")[0];
if (!"en".equals(lang) && !"default".equals(lang)) {
JSONObject json = LoadLanguageCommand.loadLanguage(this.servlet, "core", lang);
if (json != null) {
String label = json.getString("name");
writeLangData(writer, lang, label);
}
}
}
writer.endArray();
writer.endObject();
} catch (JSONException e) {
respondException(response, e);
}
}
private void writeLangData(JSONWriter writer, String lang, String label)
throws JSONException {
writer.object();
writer.key("code"); writer.value(lang);
writer.key("label"); writer.value(label);
writer.endObject();
}
}

174
main/src/com/google/refine/commands/lang/LoadLanguageCommand.java
View File

@ -1,73 +1,101 @@
package com.google.refine.commands.lang;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
public class LoadLanguageCommand extends Command {
public LoadLanguageCommand() {
// TODO Auto-generated constructor stub
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String rawDirectoryFile = request.getSession().getServletContext().getRealPath("webapp/modules/langs/");
String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "main"
+ File.separator);
BufferedReader reader = null;
String param = null;
try {
param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
} catch (NullPointerException e) {
}
if (param == null) param = request.getParameter("lng");
String[] langs = param.split(" ");
try {
String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e1) {
try {
String file = cleanedDirectory + File.separator + "translation-default.json";
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
} catch (FileNotFoundException e3) {
e3.printStackTrace();
}
}
String line = null;
String message = new String();
if (reader != null) {
while ((line = reader.readLine()) != null) {
// buffer.append(line);
message += line + System.getProperty("line.separator");
}
}
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.getWriter().println(message);
response.getWriter().flush();
response.getWriter().close();
}
}
package com.google.refine.commands.lang;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.commands.Command;
import com.google.refine.preference.PreferenceStore;
import edu.mit.simile.butterfly.ButterflyModule;
public class LoadLanguageCommand extends Command {
public LoadLanguageCommand() {
super();
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String modname = request.getParameter("module");
if (modname == null) {
modname = "core";
}
String[] langs = request.getParameterValues("lang");
if (langs == null || "".equals(langs[0])) {
PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
if (ps != null) {
langs = new String[] {(String) ps.get("userLang")};
}
}
// TODO: Switch this to just use English as the default language so we
// so we don't have to maintain a separate redundant file.
langs = Arrays.copyOf(langs, langs.length+1);
langs[langs.length-1] = "default";
JSONObject json = null;
boolean loaded = false;
for (String lang : langs) {
json = loadLanguage(this.servlet, modname, lang);
if (json != null) {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
try {
json.write(response.getWriter());
} catch (JSONException e) {
logger.error("Error writing language labels to response stream");
}
response.getWriter().flush();
response.getWriter().close();
loaded = true;
break;
}
}
if (!loaded) {
logger.error("Failed to load any language files");
}
}
static JSONObject loadLanguage(RefineServlet servlet, String modname, String lang) throws UnsupportedEncodingException {
ButterflyModule module = servlet.getModule(modname);
JSONObject json = null;
File langFile = new File(module.getPath(), "langs" + File.separator + "translation-" + lang + ".json");
try {
Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(langFile), "UTF-8"));
json = new JSONObject(new JSONTokener(reader));
} catch (FileNotFoundException e1) {
// Could be normal if we've got a list of languages as fallbacks
} catch (JSONException e) {
logger.error("JSON error reading/writing language file: " + langFile, e);
}
return json;
}
}

33
main/src/com/google/refine/commands/lang/SetLanguageCommand.java
View File

@ -1,33 +0,0 @@
package com.google.refine.commands.lang;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.preference.PreferenceStore;
public class SetLanguageCommand extends Command {
public SetLanguageCommand() {
// TODO Auto-generated constructor stub
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String lang = request.getParameter("lng");
PreferenceStore pref = ProjectManager.singleton.getPreferenceStore();
pref.put("userLang", lang);
}
}

2
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -112,7 +112,7 @@ public class CreateProjectCommand extends Command {
} else {
Format formatRecord = ImportingManager.formatToRecord.get(format);
optionObj = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format);
job, job.getSelectedFileRecords(), format);
}
adjustLegacyOptions(format, parameters, optionObj);

17
main/src/com/google/refine/expr/MetaParser.java
View File

@ -40,6 +40,7 @@ import java.util.Properties;
import java.util.Set;
import clojure.lang.IFn;
import clojure.lang.RT;
import com.google.refine.grel.Parser;
@ -58,6 +59,10 @@ abstract public class MetaParser {
}
static final protected Map<String, LanguageInfo> s_languages = new HashMap<String, LanguageInfo>();
// TODO: We should switch from using the internal compiler class
// final static private Var CLOJURE_READ_STRING = RT.var("clojure.core", "read-string");
// final static private Var CLOJURE_EVAL = RT.var("clojure.core", "eval");
static {
registerLanguageParser("grel", "Google Refine Expression Language (GREL)", new LanguageSpecificParser() {
@ -73,9 +78,17 @@ abstract public class MetaParser {
@Override
public Evaluable parse(String s) throws ParsingException {
try {
// RT.load("clojure/core"); // Make sure RT is initialized
Object foo = RT.CURRENT_NS; // Make sure RT is initialized
IFn fn = (IFn) clojure.lang.Compiler.load(new StringReader(
"(fn [value cell cells row rowIndex] " + s + ")"
));
"(fn [value cell cells row rowIndex] " + s + ")"
));
// TODO: We should to switch from using Compiler.load
// because it's technically an internal interface
// Object code = CLOJURE_READ_STRING.invoke(
// "(fn [value cell cells row rowIndex] " + s + ")"
// );
return new Evaluable() {
private IFn _fn;

66
main/src/com/google/refine/expr/functions/ToDate.java
View File

@ -33,12 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONWriter;
@ -99,29 +102,56 @@ public class ToDate implements Function {
}
// "o, format1, format2 (optional), ..."
Locale locale = Locale.getDefault();
if (args.length>=2) {
for (int i=1;i<args.length;i++) {
if (!(args[i] instanceof String)) {
// skip formats that aren't strings
continue;
}
String format = (String) args[i];
SimpleDateFormat formatter;
String format = StringUtils.trim((String) args[i]);
DateFormat formatter;
// Attempt to parse first string as a language tag
if (i == 1) {
// Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+ only
Locale possibleLocale;
int c = format.indexOf('_');
if (c > 0) {
possibleLocale = new Locale(format.substring(0, c),format.substring(c+1));
} else {
possibleLocale = new Locale(format);
}
boolean valid = false;
for (Locale l : DateFormat.getAvailableLocales()) {
if (l.equals(possibleLocale)) {
locale = possibleLocale;
valid = true;
break;
}
}
if (valid) { // If we got a valid locale
if (args.length == 2) { // No format strings to try, process using default
formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
formatter.setLenient(true);
GregorianCalendar date = parse(o1, formatter);
if (date != null) {
return date;
} else {
return new EvalError("Unable to parse as date");
}
}
continue; // Don't try to process locale string as a format string if it was valid
}
}
try {
formatter = new SimpleDateFormat(format);
formatter = new SimpleDateFormat(format,locale);
} catch (IllegalArgumentException e) {
return new EvalError("Unknown date format");
}
Date date = null;
try {
date = formatter.parse(o1);
} catch (java.text.ParseException e) {
continue;
}
formatter.setLenient(true);
GregorianCalendar date = parse(o1, formatter);
if (date != null) {
GregorianCalendar c = new GregorianCalendar();
c.setTime(date);
return c;
return date;
}
}
return new EvalError("Unable to parse as date");
@ -131,6 +161,18 @@ public class ToDate implements Function {
}
private GregorianCalendar parse(String o1, DateFormat formatter) {
try {
Date date = formatter.parse(o1);
GregorianCalendar c = new GregorianCalendar();
c.setTime(date);
return c;
} catch (java.text.ParseException e) {
return null;
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {

16
main/src/com/google/refine/expr/functions/ToString.java
View File

@ -52,23 +52,23 @@ public class ToString implements Function {
public Object call(Properties bindings, Object[] args) {
if (args.length >= 1) {
Object o1 = args[0];
if (args.length == 2) {
if (args.length == 2 && args[1] instanceof String) {
Object o2 = args[1];
if (o1 instanceof Calendar || o1 instanceof Date) {
Object o2 = args[1];
if (o2 instanceof String) {
DateFormat formatter = new SimpleDateFormat((String) o2);
return formatter.format(o1 instanceof Date ? ((Date) o1) : ((Calendar) o1).getTime());
}
DateFormat formatter = new SimpleDateFormat((String) o2);
return formatter.format(o1 instanceof Date ? ((Date) o1) : ((Calendar) o1).getTime());
} else if (o1 instanceof Number) {
return String.format((String) o2, (Number) o1);
}
} else if (args.length == 1) {
if (o1 instanceof String) {
return (String) o1;
} else if (o1 != null) {
} else {
return StringUtils.toString(o1);
}
}
}
return new EvalError("ToString accepts an object an optional second argument containing a date format string");
return new EvalError("ToString accepts an object and an optional second argument containing a date format string");
}

13
main/src/com/google/refine/expr/functions/booleans/And.java
View File

@ -38,18 +38,23 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
public class And implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
for (Object o : args) {
if (!Not.objectToBoolean(o)) {
return false;
if (args.length == 2 && args[0] instanceof Boolean && args[1] instanceof Boolean) {
for (Object o : args) {
if (!Not.objectToBoolean(o)) {
return false;
}
}
return true;
}
return true;
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects two booleans");
}
@Override

2
main/src/com/google/refine/expr/functions/booleans/Not.java
View File

@ -46,7 +46,7 @@ public class Not implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 1) {
if (args.length == 1 && args[0] instanceof Boolean) {
return !objectToBoolean(args[0]);
}
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a boolean");

13
main/src/com/google/refine/expr/functions/booleans/Or.java
View File

@ -38,18 +38,23 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
public class Or implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
for (Object o : args) {
if (Not.objectToBoolean(o)) {
return true;
if (args.length == 2 && args[0] instanceof Boolean && args[1] instanceof Boolean) {
for (Object o : args) {
if (Not.objectToBoolean(o)) {
return true;
}
}
return false;
}
return false;
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects two booleans");
}
@Override

60
main/src/com/google/refine/expr/functions/booleans/Xor.java
View File

@ -0,0 +1,60 @@
/*
Copyright (c) 2013, Jesus M. Castagnetto
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.expr.functions.booleans;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
public class Xor implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 2 && args[0] instanceof Boolean && args[1] instanceof Boolean) {
return (Boolean) args[0] ^ (Boolean) args[1];
}
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 booleans");
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description");
writer.value("XORs two boolean values");
writer.key("params");
writer.value("boolean a, boolean b");
writer.key("returns");
writer.value("boolean");
writer.endObject();
}
}

2
main/src/com/google/refine/expr/functions/date/DatePart.java
View File

@ -75,6 +75,8 @@ public class DatePart implements Function {
return c.get(Calendar.MINUTE);
} else if ("seconds".equals(part) || "sec".equals(part) || "s".equals(part)) {
return c.get(Calendar.SECOND);
} else if ("milliseconds".equals(part) || "ms".equals(part) || "S".equals(part)) {
return c.get(Calendar.MILLISECOND);
} else if ("years".equals(part) || "year".equals(part)) {
return c.get(Calendar.YEAR);
} else if ("months".equals(part) || "month".equals(part)) { // avoid 'm' to avoid confusion with minute

2
main/src/com/google/refine/expr/functions/math/Floor.java
View File

@ -57,7 +57,7 @@ public class Floor implements Function {
throws JSONException {
writer.object();
writer.key("description"); writer.value("Returns the floor of a number");
writer.key("description"); writer.value("Returns the floor of a number as an integer");
writer.key("params"); writer.value("number d");
writer.key("returns"); writer.value("number");
writer.endObject();

2
main/src/com/google/refine/grel/ControlFunctionRegistry.java
View File

@ -56,6 +56,7 @@ import com.google.refine.expr.functions.arrays.Uniques;
import com.google.refine.expr.functions.booleans.And;
import com.google.refine.expr.functions.booleans.Not;
import com.google.refine.expr.functions.booleans.Or;
import com.google.refine.expr.functions.booleans.Xor;
import com.google.refine.expr.functions.date.DatePart;
import com.google.refine.expr.functions.date.Inc;
import com.google.refine.expr.functions.date.Now;
@ -284,6 +285,7 @@ public class ControlFunctionRegistry {
registerFunction("and", new And());
registerFunction("or", new Or());
registerFunction("not", new Not());
registerFunction("xor", new Xor());
registerFunction("cross", new Cross());

18
main/src/com/google/refine/grel/controls/Filter.java
View File

@ -81,7 +81,11 @@ public class Filter implements Control {
results = new ArrayList<Object>(values.length);
for (Object v : values) {
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
@ -97,7 +101,11 @@ public class Filter implements Control {
try {
Object v = a.get(i);
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
@ -113,7 +121,11 @@ public class Filter implements Control {
results = new ArrayList<Object>(collection.size());
for (Object v : collection) {
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
if (r instanceof Boolean && ((Boolean) r).booleanValue()) {

18
main/src/com/google/refine/grel/controls/ForEach.java
View File

@ -81,7 +81,11 @@ public class ForEach implements Control {
results = new ArrayList<Object>(values.length);
for (Object v : values) {
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
@ -96,7 +100,11 @@ public class ForEach implements Control {
try {
Object v = a.get(i);
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
@ -111,7 +119,11 @@ public class ForEach implements Control {
results = new ArrayList<Object>(collection.size());
for (Object v : collection) {
bindings.put(name, v);
if (v != null) {
bindings.put(name, v);
} else {
bindings.remove(name);
}
Object r = args[2].evaluate(bindings);

15
main/src/com/google/refine/history/HistoryEntry.java
View File

@ -40,6 +40,8 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
@ -53,6 +55,7 @@ import com.google.refine.util.ParsingUtilities;
* obtain information about a change without actually loading the change.
*/
public class HistoryEntry implements Jsonizable {
final static Logger logger = LoggerFactory.getLogger("HistoryEntry");
final public long id;
final public long projectID;
final public String description;
@ -82,13 +85,7 @@ public class HistoryEntry implements Jsonizable {
}
public HistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change) {
this.id = id;
this.projectID = project.id;
this.description = description;
this.operation = operation;
this.time = new Date();
this._manager = ProjectManager.singleton.getHistoryEntryManager();
this(id,project.id,description,operation,new Date());
setChange(change);
}
@ -99,6 +96,10 @@ public class HistoryEntry implements Jsonizable {
this.operation = operation;
this.time = time;
this._manager = ProjectManager.singleton.getHistoryEntryManager();
if (this._manager == null) {
logger.error("Failed to get history entry manager from project manager: "
+ ProjectManager.singleton );
}
}
@Override

16
main/src/com/google/refine/importers/BinaryFormatGuesser.java Normal file
View File

@ -0,0 +1,16 @@
package com.google.refine.importers;
import java.io.File;
import com.google.refine.importing.FormatGuesser;
public class BinaryFormatGuesser implements FormatGuesser {
@Override
public String guess(File file, String encoding, String seedFormat) {
// TODO: Guess based on sniffing magic numbers
return null;
}
}

19
main/src/com/google/refine/importers/ExcelImporter.java
View File

@ -43,6 +43,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.poi.POIXMLException;
import org.apache.poi.common.usermodel.Hyperlink;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@ -115,6 +116,10 @@ public class ExcelImporter extends TabularImportingParserBase {
}
} catch (IOException e) {
logger.error("Error generating parser UI initialization data for Excel file", e);
} catch (IllegalArgumentException e) {
logger.error("Error generating parser UI initialization data for Excel file (only Excel 97 & later supported)", e);
} catch (POIXMLException e) {
logger.error("Error generating parser UI initialization data for Excel file - invalid XML", e);
}
return options;
@ -153,6 +158,20 @@ public class ExcelImporter extends TabularImportingParserBase {
e
));
return;
} catch (IllegalArgumentException e) {
exceptions.add(new ImportException(
"Attempted to parse as an Excel file but failed. " +
"Only Excel 97 and later formats are supported.",
e
));
return;
} catch (POIXMLException e) {
exceptions.add(new ImportException(
"Attempted to parse as an Excel file but failed. " +
"Invalid XML.",
e
));
return;
}
int[] sheets = JSONUtilities.getIntArray(options, "sheets");

7
main/src/com/google/refine/importers/FixedWidthImporter.java
View File

@ -46,7 +46,7 @@ public class FixedWidthImporter extends TabularImportingParserBase {
JSONUtilities.safePut(options, "headerLines", 0);
JSONUtilities.safePut(options, "columnWidths", columnWidths);
JSONUtilities.safePut(options, "guessCellValueTypes", true);
JSONUtilities.safePut(options, "guessCellValueTypes", false);
}
return options;
}
@ -111,9 +111,8 @@ public class FixedWidthImporter extends TabularImportingParserBase {
/**
* Splits the line into columns
* @param line
* @param lnReader
* @param splitIntoColumns
* @param line Line to be split
* @param widths array of integers with field sizes
* @return
*/
static private ArrayList<Object> getCells(String line, int[] widths) {

6
main/src/com/google/refine/importers/ImporterUtilities.java
View File

@ -209,10 +209,8 @@ public class ImporterUtilities {
long totalBytesRead = 0;
void setProgress(String fileSource, long bytesRead) {
ImportingUtilities.setCreatingProjectProgress(
job,
"Reading " + fileSource,
totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2));
job.setProgress(totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2),
"Reading " + fileSource);
}
@Override

2
main/src/com/google/refine/importers/LineBasedImporter.java
View File

@ -29,7 +29,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
JSONUtilities.safePut(options, "linesPerRow", 1);
JSONUtilities.safePut(options, "headerLines", 0);
JSONUtilities.safePut(options, "guessCellValueTypes", true);
JSONUtilities.safePut(options, "guessCellValueTypes", false);
return options;
}

106
main/src/com/google/refine/importers/MarcImporter.java
View File

@ -35,12 +35,10 @@ package com.google.refine.importers;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import org.json.JSONObject;
import org.marc4j.MarcPermissiveStreamReader;
@ -48,62 +46,58 @@ import org.marc4j.MarcWriter;
import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record;
import com.google.refine.ProjectMetadata;
import com.google.refine.importers.tree.ImportColumnGroup;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.importing.ImportingUtilities;
import com.google.refine.util.JSONUtilities;
public class MarcImporter extends XmlImporter {
@Override
public void parseOneFile(Project project, ProjectMetadata metadata,
ImportingJob job, String fileSource, InputStream inputStream,
ImportColumnGroup rootColumnGroup, int limit, JSONObject options,
List<Exception> exceptions) {
File tempFile;
try {
tempFile = File.createTempFile("refine-import-", ".marc.xml");
} catch (IOException e) {
exceptions.add(new ImportException("Unexpected error creating temp file", e));
return;
}
try {
OutputStream os = new FileOutputStream(tempFile);
try {
MarcWriter writer = new MarcXmlWriter(os, true);
MarcPermissiveStreamReader reader = new MarcPermissiveStreamReader(
inputStream, true, true);
while (reader.hasNext()) {
Record record = reader.next();
writer.write(record);
}
writer.close();
} finally {
try {
os.close();
} catch (IOException e) {
// Just ignore - not much we can do anyway
}
}
InputStream is = new FileInputStream(tempFile);
try {
super.parseOneFile(project, metadata, job, fileSource, inputStream,
rootColumnGroup, limit, options, exceptions);
} finally {
try {
is.close();
} catch (IOException e) {
// Just ignore - not much we can do anyway
}
}
} catch (FileNotFoundException e) {
exceptions.add(new ImportException("Input file not found", e));
return;
} finally {
tempFile.delete();
}
public MarcImporter() {
super();
}
@Override
public JSONObject createParserUIInitializationData(ImportingJob job, java.util.List<JSONObject> fileRecords, String format) {
if (fileRecords.size() > 0) {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
File tempFile = new File(file.getAbsolutePath()+".xml");
JSONUtilities.safePut(firstFileRecord, "location",
JSONUtilities.getString(firstFileRecord, "location", "")+".xml");
try {
InputStream inputStream = new FileInputStream(file);
OutputStream outputStream = new FileOutputStream(tempFile);
try {
MarcWriter writer = new MarcXmlWriter(outputStream, true);
MarcPermissiveStreamReader reader = new MarcPermissiveStreamReader(
inputStream, true, true);
while (reader.hasNext()) {
Record record = reader.next();
writer.write(record);
}
writer.close();
} finally {
try {
outputStream.close();
inputStream.close();
file.delete(); // get rid of our original file
} catch (IOException e) {
// Just ignore - not much we can do anyway
}
}
} catch (IOException e) {
logger.error("Failed to create temporary XML file from MARC file", e);
}
}
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
return options;
};
}

29
main/src/com/google/refine/importers/RdfTripleImporter.java
View File

@ -86,18 +86,23 @@ public class RdfTripleImporter extends ImportingParserBase {
JSONObject options, List<Exception> exceptions) {
Graph graph;
switch (mode) {
case NT:
graph = rdfReader.parseNTriples(input);
break;
case N3:
graph = rdfReader.parseN3(input);
break;
case RDFXML:
graph = rdfReader.parseRdfXml(input);
break;
default:
throw new IllegalArgumentException("Unknown parsing mode");
try {
switch (mode) {
case NT:
graph = rdfReader.parseNTriples(input);
break;
case N3:
graph = rdfReader.parseN3(input);
break;
case RDFXML:
graph = rdfReader.parseRdfXml(input);
break;
default:
throw new IllegalArgumentException("Unknown parsing mode");
}
} catch (Exception e) {
exceptions.add(e);
return;
}
ClosableIterable<Triple> triples = graph.find(ANY_SUBJECT_NODE, ANY_PREDICATE_NODE, ANY_OBJECT_NODE);

33
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -73,7 +73,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
String separator = guessSeparator(job, fileRecords);
JSONUtilities.safePut(options, "separator", separator != null ? separator : "\\t");
JSONUtilities.safePut(options, "guessCellValueTypes", true);
JSONUtilities.safePut(options, "guessCellValueTypes", false);
JSONUtilities.safePut(options, "processQuotes", true);
return options;
@ -99,9 +99,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
boolean strictQuotes = JSONUtilities.getBoolean(options, "strictQuotes", false);
final CSVParser parser = new CSVParser(
sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators.
sep,
CSVParser.DEFAULT_QUOTE_CHARACTER,
(char) 127, // we don't want escape processing try DEL as a rare character until we can turn it off
(char) 0, // we don't want escape processing
strictQuotes,
CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE,
!processQuotes);
@ -168,6 +168,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
return guessSeparator(file, encoding, false); // quotes off for backward compatibility
}
// TODO: Move this to the CSV project?
static public Separator guessSeparator(File file, String encoding, boolean handleQuotes) {
try {
InputStream is = new FileInputStream(file);
@ -190,7 +191,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
if (s.length() == 0) {
continue;
}
lineCount++;
if (!inQuote) {
lineCount++;
}
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
@ -212,10 +215,12 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
}
}
for (Separator separator : separators) {
separator.totalCount += separator.currentLineCount;
separator.totalOfSquaredCount += separator.currentLineCount * separator.currentLineCount;
separator.currentLineCount = 0;
if (!inQuote) {
for (Separator separator : separators) {
separator.totalCount += separator.currentLineCount;
separator.totalOfSquaredCount += separator.currentLineCount * separator.currentLineCount;
separator.currentLineCount = 0;
}
}
}
@ -231,14 +236,16 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
Collections.sort(separators, new Comparator<Separator>() {
@Override
public int compare(Separator sep0, Separator sep1) {
return Double.compare(sep0.stddev, sep1.stddev);
return Double.compare(sep0.stddev / sep0.averagePerLine,
sep1.stddev / sep1.averagePerLine);
}
});
for (Separator separator : separators) {
if (separator.stddev / separator.averagePerLine < 0.1) {
return separator;
}
Separator separator = separators.get(0);
if (separator.stddev / separator.averagePerLine < 0.1) {
return separator;
}
}
} finally {
lineNumberReader.close();

2
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -228,7 +228,7 @@ public class DefaultImportingController implements ImportingController {
Format formatRecord = ImportingManager.formatToRecord.get(format);
if (formatRecord != null && formatRecord.parser != null) {
JSONObject options = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format);
job, job.getSelectedFileRecords(), format);
JSONObject result = new JSONObject();
JSONUtilities.safePut(result, "status", "ok");
JSONUtilities.safePut(result, "options", options);

110
main/src/com/google/refine/importing/ImportingJob.java
View File

@ -35,14 +35,18 @@ package com.google.refine.importing;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.model.Project;
import com.google.refine.util.JSONUtilities;
@ -52,7 +56,7 @@ public class ImportingJob implements Jsonizable {
final public long id;
final public File dir; // Temporary directory where the data about this job is stored
public JSONObject config = null;
private JSONObject config;
public Project project;
public ProjectMetadata metadata;
@ -61,22 +65,101 @@ public class ImportingJob implements Jsonizable {
public boolean updating;
public boolean canceled;
final private Object lock = new Object();
public ImportingJob(long id, File dir) {
this.id = id;
this.dir = dir;
JSONObject cfg = new JSONObject();
JSONUtilities.safePut(cfg, "state", "new");
JSONUtilities.safePut(cfg, "hasData", false);
this.config = cfg;
dir.mkdirs();
}
public JSONObject getOrCreateDefaultConfig() {
if (config == null) {
config = new JSONObject();
JSONUtilities.safePut(config, "state", "new");
JSONUtilities.safePut(config, "hasData", false);
}
return config;
}
public void setState(String state) {
synchronized(config) {
JSONUtilities.safePut(config, "state", state);
}
}
public void setError(List<Exception> exceptions) {
synchronized(config) {
JSONUtilities.safePut(config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
setState("error");
}
}
public void setProjectID(long projectID) {
synchronized (config) {
JSONUtilities.safePut(config, "projectID", projectID);
}
}
public void setProgress(int percent, String message) {
synchronized (config) {
JSONObject progress = JSONUtilities.getObject(config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
}
public void setFileSelection(JSONArray fileSelectionArray) {
synchronized (config) {
JSONUtilities.safePut(config, "fileSelection", fileSelectionArray);
}
}
public void setRankedFormats(JSONArray rankedFormats) {
synchronized (config) {
JSONUtilities.safePut(config, "rankedFormats", rankedFormats);
}
}
public JSONObject getRetrievalRecord() {
synchronized(config) {
return JSONUtilities.getObject(config,"retrievalRecord");
}
}
public List<JSONObject> getSelectedFileRecords() {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
public void touch() {
lastTouched = System.currentTimeMillis();
}
@ -85,6 +168,11 @@ public class ImportingJob implements Jsonizable {
if (project != null) {
project.dispose();
}
// Make sure all projects have been saved in case we run out of memory
// or have some other catastrophe on import
ProjectManager.singleton.save(true);
project = new Project();
metadata = new ProjectMetadata();
}
@ -111,8 +199,12 @@ public class ImportingJob implements Jsonizable {
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
synchronized(lock) {
writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
}
}
}

41
main/src/com/google/refine/importing/ImportingManager.java
View File

@ -35,6 +35,9 @@ package com.google.refine.importing;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@ -42,8 +45,9 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.json.JSONException;
@ -82,7 +86,7 @@ public class ImportingManager {
static private RefineServlet servlet;
static private File importDir;
final static private Map<Long, ImportingJob> jobs = new HashMap<Long, ImportingJob>();
final static private Map<Long, ImportingJob> jobs = Collections.synchronizedMap(new HashMap<Long, ImportingJob>());
// Mapping from format to label, e.g., "text" to "Text files", "text/xml" to "XML files"
final static public Map<String, Format> formatToRecord = new HashMap<String, Format>();
@ -103,29 +107,25 @@ public class ImportingManager {
final static public Map<String, ImportingController> controllers = new HashMap<String, ImportingController>();
// timer for periodically deleting stale importing jobs
static private Timer _timer;
static private ScheduledExecutorService service;
final static private long s_timerPeriod = 1000 * 60 * 10; // 10 minutes
final static private long s_stalePeriod = 1000 * 60 * 60; // 60 minutes
final static private long TIMER_PERIOD = 10; // 10 minutes
final static private long STALE_PERIOD = 60; // 60 minutes
static private class CleaningTimerTask extends TimerTask {
static private class CleaningTimerTask implements Runnable {
@Override
public void run() {
try {
cleanUpStaleJobs();
} finally {
_timer.schedule(new CleaningTimerTask(), s_timerPeriod);
// we don't use scheduleAtFixedRate because that might result in
// bunched up events when the computer is put in sleep mode
}
// An exception here will keep future runs of this task from happening,
// but won't affect other timer tasks
cleanUpStaleJobs();
}
}
static public void initialize(RefineServlet servlet) {
ImportingManager.servlet = servlet;
_timer = new Timer("autosave");
_timer.schedule(new CleaningTimerTask(), s_timerPeriod);
service = Executors.newSingleThreadScheduledExecutor();
service.scheduleWithFixedDelay(new CleaningTimerTask(), TIMER_PERIOD, TIMER_PERIOD, TimeUnit.MINUTES);
}
static public void registerFormat(String format, String label) {
@ -288,12 +288,15 @@ public class ImportingManager {
static private void cleanUpStaleJobs() {
long now = System.currentTimeMillis();
for (Long id : new HashSet<Long>(jobs.keySet())) {
Collection<Long> keys;
synchronized(jobs) {
keys = new ArrayList<Long>(jobs.keySet());
}
for (Long id : keys) {
ImportingJob job = jobs.get(id);
if (job != null && !job.updating && now - job.lastTouched > s_stalePeriod) {
if (job != null && !job.updating && now - job.lastTouched > STALE_PERIOD) {
job.dispose();
jobs.remove(id);
logger.info("Disposed " + id);
}
}

245
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -42,9 +42,9 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -65,6 +65,14 @@ import org.apache.commons.fileupload.ProgressListener;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.fileupload.util.Streams;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DecompressingHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
@ -73,8 +81,6 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.ibm.icu.text.NumberFormat;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.RefineServlet;
@ -147,14 +153,14 @@ public class ImportingUtilities {
}
static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) {
JSONUtilities.safePut(job.config, "fileSelection", fileSelectionArray);
job.setFileSelection(fileSelectionArray);
String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray);
bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
JSONArray rankedFormats = new JSONArray();
JSONUtilities.safePut(job.config, "rankedFormats", rankedFormats);
ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
job.setRankedFormats(rankedFormats);
}
static public void retrieveContentFromPostRequest(
@ -210,15 +216,15 @@ public class ImportingUtilities {
}
});
List tempFiles = upload.parseRequest(request);
@SuppressWarnings("unchecked")
List<FileItem> tempFiles = (List<FileItem>)upload.parseRequest(request);
progress.setProgress("Uploading data ...", -1);
parts: for (Object obj : tempFiles) {
parts: for (FileItem fileItem : tempFiles) {
if (progress.isCanceled()) {
break;
}
FileItem fileItem = (FileItem) obj;
InputStream stream = fileItem.getInputStream();
String name = fileItem.getFieldName().toLowerCase();
@ -243,10 +249,10 @@ public class ImportingUtilities {
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
JSONUtilities.append(fileRecords, fileRecord);
clipboardCount++;
JSONUtilities.append(fileRecords, fileRecord);
} else if (name.equals("download")) {
String urlString = Streams.asString(stream);
URL url = new URL(urlString);
@ -270,56 +276,77 @@ public class ImportingUtilities {
}
}
}
URLConnection urlConnection = url.openConnection();
urlConnection.setConnectTimeout(5000);
if (urlConnection instanceof HttpURLConnection) {
HttpURLConnection httpConnection = (HttpURLConnection) urlConnection;
RefineServlet.setUserAgent(httpConnection);
}
// TODO: Set Accept-Encoding on connection so we don't get stuff we can't handle?
urlConnection.connect();
InputStream stream2 = urlConnection.getInputStream();
try {
String localname = url.getPath();
if (localname.isEmpty() || localname.endsWith("/")) {
localname = localname + "temp";
}
File file = allocateFile(rawDataDir, localname);
int contentLength = urlConnection.getContentLength();
if (contentLength > 0) {
update.totalExpectedSize += contentLength;
}
JSONUtilities.safePut(fileRecord, "declaredEncoding", urlConnection.getContentEncoding());
JSONUtilities.safePut(fileRecord, "declaredMimeType", urlConnection.getContentType());
JSONUtilities.safePut(fileRecord, "fileName", file.getName());
JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
progress.setProgress("Downloading " + urlString,
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
long actualLength = saveStreamToFile(stream2, file, update);
JSONUtilities.safePut(fileRecord, "size", actualLength);
if (actualLength == 0) {
throw new Exception("No content found in " + urlString);
} else if (contentLength >= 0) {
update.totalExpectedSize += (actualLength - contentLength);
} else {
update.totalExpectedSize += actualLength;
if ("http".equals(url.getProtocol()) || "https".equals(url.getProtocol())) {
DefaultHttpClient client = new DefaultHttpClient();
DecompressingHttpClient httpclient =
new DecompressingHttpClient(client);
HttpGet httpGet = new HttpGet(url.toURI());
httpGet.setHeader("User-Agent", RefineServlet.getUserAgent());
if ("https".equals(url.getProtocol())) {
// HTTPS only - no sending password in the clear over HTTP
String userinfo = url.getUserInfo();
if (userinfo != null) {
int s = userinfo.indexOf(':');
if (s > 0) {
String user = userinfo.substring(0, s);
String pw = userinfo.substring(s + 1, userinfo.length());
client.getCredentialsProvider().setCredentials(
new AuthScope(url.getHost(), 443),
new UsernamePasswordCredentials(user, pw));
}
}
}
progress.setProgress("Saving " + urlString + " locally",
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
HttpResponse response = httpclient.execute(httpGet);
if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
archiveCount++;
try {
response.getStatusLine();
HttpEntity entity = response.getEntity();
if (entity == null) {
throw new Exception("No content found in " + url.toString());
}
InputStream stream2 = entity.getContent();
String encoding = null;
if (entity.getContentEncoding() != null) {
encoding = entity.getContentEncoding().getValue();
}
JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
String contentType = null;
if (entity.getContentType().getValue() != null) {
contentType = entity.getContentType().getValue();
}
JSONUtilities.safePut(fileRecord, "declaredMimeType", contentType);
if (saveStream(stream2, url, rawDataDir, progress, update,
fileRecord, fileRecords,
entity.getContentLength())) {
archiveCount++;
}
downloadCount++;
EntityUtils.consume(entity);
} finally {
httpGet.releaseConnection();
}
} else {
// Fallback handling for non HTTP connections (only FTP?)
URLConnection urlConnection = url.openConnection();
urlConnection.setConnectTimeout(5000);
urlConnection.connect();
InputStream stream2 = urlConnection.getInputStream();
JSONUtilities.safePut(fileRecord, "declaredEncoding",
urlConnection.getContentEncoding());
JSONUtilities.safePut(fileRecord, "declaredMimeType",
urlConnection.getContentType());
try {
if (saveStream(stream2, url, rawDataDir, progress,
update, fileRecord, fileRecords,
urlConnection.getContentLength())) {
archiveCount++;
}
downloadCount++;
} finally {
stream2.close();
}
downloadCount++;
} finally {
stream2.close();
}
} else {
String value = Streams.asString(stream);
@ -360,8 +387,8 @@ public class ImportingUtilities {
}
// Delete all temp files.
for (Object obj : tempFiles) {
((FileItem)obj).delete();
for (FileItem fileItem : tempFiles) {
fileItem.delete();
}
JSONUtilities.safePut(retrievalRecord, "uploadCount", uploadCount);
@ -369,6 +396,37 @@ public class ImportingUtilities {
JSONUtilities.safePut(retrievalRecord, "clipboardCount", clipboardCount);
JSONUtilities.safePut(retrievalRecord, "archiveCount", archiveCount);
}
private static boolean saveStream(InputStream stream, URL url, File rawDataDir, final Progress progress,
final SavingUpdate update, JSONObject fileRecord, JSONArray fileRecords, long length)
throws IOException, Exception {
String localname = url.getPath();
if (localname.isEmpty() || localname.endsWith("/")) {
localname = localname + "temp";
}
File file = allocateFile(rawDataDir, localname);
JSONUtilities.safePut(fileRecord, "fileName", file.getName());
JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
update.totalExpectedSize += length;
progress.setProgress("Downloading " + url.toString(),
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
long actualLength = saveStreamToFile(stream, file, update);
JSONUtilities.safePut(fileRecord, "size", actualLength);
if (actualLength == 0) {
throw new Exception("No content found in " + url.toString());
} else if (length >= 0) {
update.totalExpectedSize += (actualLength - length);
} else {
update.totalExpectedSize += actualLength;
}
progress.setProgress("Saving " + url.toString() + " locally",
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
return postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress);
}
static public String getRelativePath(File file, File dir) {
String location = file.getAbsolutePath().substring(dir.getAbsolutePath().length());
@ -626,17 +684,13 @@ public class ImportingUtilities {
static public InputStream tryOpenAsCompressedFile(File file, String mimeType, String contentEncoding) {
String fileName = file.getName();
try {
/*
* TODO: Do we need to support MIME types as well as content encodings?
* application/x-bzip2
* application/x-gzip
* multipart/x-gzip
*/
if (fileName.endsWith(".gz")
|| "gzip".equals(contentEncoding)
|| "x-gzip".equals(contentEncoding)) {
|| "x-gzip".equals(contentEncoding)
|| "application/x-gzip".equals(mimeType)) {
return new GZIPInputStream(new FileInputStream(file));
} else if (fileName.endsWith(".bz2")) {
} else if (fileName.endsWith(".bz2")
||"application/x-bzip2".equals(mimeType)) {
InputStream is = new FileInputStream(file);
is.mark(4);
if (!(is.read() == 'B' && is.read() == 'Z')) {
@ -693,6 +747,15 @@ public class ImportingUtilities {
return encoding;
}
/**
* Figure out the best (most common) format for the set of files, select
* all files which match that format, and return the format found.
*
* @param job ImportingJob object
* @param retrievalRecord JSON object containing "files" key with all our files
* @param fileSelectionIndexes JSON array of selected file indices matching best format
* @return best (highest frequency) format
*/
static public String autoSelectFiles(ImportingJob job, JSONObject retrievalRecord, JSONArray fileSelectionIndexes) {
final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
List<String> formats = new ArrayList<String>();
@ -747,7 +810,7 @@ public class ImportingUtilities {
}
static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
JSONObject retrievalRecord = job.getRetrievalRecord();
final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
List<String> formats = new ArrayList<String>();
@ -780,7 +843,7 @@ public class ImportingUtilities {
}
static String guessBetterFormat(ImportingJob job, String bestFormat) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
JSONObject retrievalRecord = job.getRetrievalRecord();
return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat;
}
@ -879,27 +942,7 @@ public class ImportingUtilities {
JSONUtilities.append(rankedFormats, format);
}
}
static public List<JSONObject> getSelectedFileRecords(ImportingJob job) {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(job.config, "fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) {
Format record = ImportingManager.formatToRecord.get(format);
@ -914,7 +957,7 @@ public class ImportingUtilities {
job.project,
job.metadata,
job,
getSelectedFileRecords(job),
job.getSelectedFileRecords(),
format,
100,
optionObj,
@ -936,7 +979,7 @@ public class ImportingUtilities {
return -1;
}
JSONUtilities.safePut(job.config, "state", "creating-project");
job.setState("creating-project");
final Project project = new Project();
if (synchronous) {
@ -975,7 +1018,7 @@ public class ImportingUtilities {
project,
pm,
job,
getSelectedFileRecords(job),
job.getSelectedFileRecords(),
format,
-1,
optionObj,
@ -988,27 +1031,15 @@ public class ImportingUtilities {
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id);
JSONUtilities.safePut(job.config, "state", "created-project");
job.setProjectID(project.id);
job.setState("created-project");
} else {
JSONUtilities.safePut(job.config, "state", "error");
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
job.setError(exceptions);
}
job.touch();
job.updating = false;
}
}
static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
}

68
main/src/com/google/refine/io/FileProjectManager.java
View File

@ -62,18 +62,20 @@ import com.google.refine.model.Project;
import com.google.refine.preference.TopList;
public class FileProjectManager extends ProjectManager {
final static protected String s_projectDirNameSuffix = ".project";
final static protected String PROJECT_DIR_SUFFIX = ".project";
protected File _workspaceDir;
final static Logger logger = LoggerFactory.getLogger("FileProjectManager");
static public synchronized void initialize(File dir) {
if (singleton == null) {
logger.info("Using workspace directory: {}", dir.getAbsolutePath());
singleton = new FileProjectManager(dir);
if (singleton != null) {
logger.warn("Overwriting singleton already set: " + singleton);
}
logger.info("Using workspace directory: {}", dir.getAbsolutePath());
singleton = new FileProjectManager(dir);
// This needs our singleton set, thus the unconventional control flow
((FileProjectManager) singleton).recover();
}
protected FileProjectManager(File dir) {
@ -85,7 +87,6 @@ public class FileProjectManager extends ProjectManager {
}
load();
recover();
}
public File getWorkspaceDir() {
@ -93,7 +94,7 @@ public class FileProjectManager extends ProjectManager {
}
static public File getProjectDir(File workspaceDir, long projectID) {
File dir = new File(workspaceDir, projectID + s_projectDirNameSuffix);
File dir = new File(workspaceDir, projectID + PROJECT_DIR_SUFFIX);
if (!dir.exists()) {
dir.mkdir();
}
@ -114,6 +115,9 @@ public class FileProjectManager extends ProjectManager {
public boolean loadProjectMetadata(long projectID) {
synchronized (this) {
ProjectMetadata metadata = ProjectMetadataUtilities.load(getProjectDir(projectID));
if (metadata == null) {
metadata = ProjectMetadataUtilities.recover(getProjectDir(projectID), projectID);
}
if (metadata != null) {
_projectsMetadata.put(projectID, metadata);
return true;
@ -217,7 +221,7 @@ public class FileProjectManager extends ProjectManager {
}
@Override
protected void saveProject(Project project){
protected void saveProject(Project project) throws IOException{
ProjectUtilities.save(project);
}
@ -227,7 +231,6 @@ public class FileProjectManager extends ProjectManager {
}
/**
* Save the workspace's data out to file in a safe way: save to a temporary file first
* and rename it to the real file.
@ -237,7 +240,12 @@ public class FileProjectManager extends ProjectManager {
synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json");
try {
saveToFile(tempFile);
if (!saveToFile(tempFile)) {
// If the save wasn't really needed, just keep what we had
tempFile.delete();
logger.info("Skipping unnecessary workspace save");
return;
}
} catch (Exception e) {
e.printStackTrace();
@ -248,21 +256,23 @@ public class FileProjectManager extends ProjectManager {
File file = new File(_workspaceDir, "workspace.json");
File oldFile = new File(_workspaceDir, "workspace.old.json");
if (oldFile.exists()) {
oldFile.delete();
}
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
logger.info("Saved workspace");
}
}
protected void saveToFile(File file) throws IOException, JSONException {
protected boolean saveToFile(File file) throws IOException, JSONException {
FileWriter writer = new FileWriter(file);
boolean saveWasNeeded = false;
try {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.object();
@ -272,11 +282,9 @@ public class FileProjectManager extends ProjectManager {
ProjectMetadata metadata = _projectsMetadata.get(id);
if (metadata != null) {
jsonWriter.value(id);
try {
if (metadata.isDirty()) {
ProjectMetadataUtilities.save(metadata, getProjectDir(id));
} catch (Exception e) {
e.printStackTrace();
saveWasNeeded = true;
}
}
}
@ -284,12 +292,14 @@ public class FileProjectManager extends ProjectManager {
writer.write('\n');
jsonWriter.key("preferences");
saveWasNeeded |= _preferenceStore.isDirty();
_preferenceStore.write(jsonWriter, new Properties());
jsonWriter.endObject();
} finally {
writer.close();
}
return saveWasNeeded;
}
@ -386,11 +396,12 @@ public class FileProjectManager extends ProjectManager {
}
protected void recover() {
boolean recovered = false;
for (File file : _workspaceDir.listFiles()) {
if (file.isDirectory() && !file.isHidden()) {
String name = file.getName();
if (file.getName().endsWith(s_projectDirNameSuffix)) {
String idString = name.substring(0, name.length() - s_projectDirNameSuffix.length());
String dirName = file.getName();
if (file.getName().endsWith(PROJECT_DIR_SUFFIX)) {
String idString = dirName.substring(0, dirName.length() - PROJECT_DIR_SUFFIX.length());
long id = -1;
try {
id = Long.parseLong(idString);
@ -400,19 +411,22 @@ public class FileProjectManager extends ProjectManager {
if (id > 0 && !_projectsMetadata.containsKey(id)) {
if (loadProjectMetadata(id)) {
logger.info(
"Recovered project named " +
getProjectMetadata(id).getName() +
" in directory " + name);
logger.info("Recovered project named "
+ getProjectMetadata(id).getName()
+ " in directory " + dirName);
recovered = true;
} else {
logger.warn("Failed to recover project in directory " + name);
logger.warn("Failed to recover project in directory " + dirName);
file.renameTo(new File(file.getParentFile(), name + ".corrupted"));
file.renameTo(new File(file.getParentFile(), dirName + ".corrupted"));
}
}
}
}
}
if (recovered) {
saveWorkspace();
}
}
@Override

65
main/src/com/google/refine/io/ProjectMetadataUtilities.java
View File

@ -36,9 +36,14 @@ package com.google.refine.io;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.json.JSONWriter;
@ -46,36 +51,31 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectMetadata;
import com.google.refine.model.Project;
public class ProjectMetadataUtilities {
final static Logger logger = LoggerFactory.getLogger("project_metadata_utilities");
public static void save(ProjectMetadata projectMeta, File projectDir) throws Exception {
public static void save(ProjectMetadata projectMeta, File projectDir) throws JSONException, IOException {
File tempFile = new File(projectDir, "metadata.temp.json");
try {
saveToFile(projectMeta, tempFile);
} catch (Exception e) {
e.printStackTrace();
logger.warn("Failed to save project metadata");
return;
}
saveToFile(projectMeta, tempFile);
File file = new File(projectDir, "metadata.json");
File oldFile = new File(projectDir, "metadata.old.json");
if (oldFile.exists()) {
oldFile.delete();
}
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
}
protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws Exception {
protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws JSONException, IOException {
Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
try {
JSONWriter jsonWriter = new JSONWriter(writer);
@ -103,6 +103,45 @@ public class ProjectMetadataUtilities {
return null;
}
/**
* Reconstruct the project metadata on a best efforts basis. The name is
* gone, so build something descriptive from the column names. Recover the
* creation and modification times based on whatever files are available.
*
* @param projectDir the project directory
* @param id the proejct id
* @return
*/
static public ProjectMetadata recover(File projectDir, long id) {
ProjectMetadata pm = null;
Project p = ProjectUtilities.load(projectDir, id);
if (p != null) {
List<String> columnNames = p.columnModel.getColumnNames();
String tempName = "<recovered project> - " + columnNames.size()
+ " cols X " + p.rows.size() + " rows - "
+ StringUtils.join(columnNames,'|');
p.dispose();
long ctime = System.currentTimeMillis();
long mtime = 0;
File dataFile = new File(projectDir, "data.zip");
ctime = mtime = dataFile.lastModified();
File historyDir = new File(projectDir,"history");
File[] files = historyDir.listFiles();
if (files != null) {
for (File f : files) {
long time = f.lastModified();
ctime = Math.min(ctime, time);
mtime = Math.max(mtime, time);
}
}
pm = new ProjectMetadata(new Date(ctime),new Date(mtime), tempName);
logger.error("Partially recovered missing metadata project in directory " + projectDir + " - " + tempName);
}
return pm;
}
static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception {
FileReader reader = new FileReader(metadataFile);

15
main/src/com/google/refine/io/ProjectUtilities.java
View File

@ -35,6 +35,7 @@ package com.google.refine.io;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
@ -50,7 +51,7 @@ import com.google.refine.util.Pool;
public class ProjectUtilities {
final static Logger logger = LoggerFactory.getLogger("project_utilities");
synchronized public static void save(Project project) {
synchronized public static void save(Project project) throws IOException {
synchronized (project) {
long id = project.id;
File dir = ((FileProjectManager)ProjectManager.singleton).getProjectDir(id);
@ -58,11 +59,15 @@ public class ProjectUtilities {
File tempFile = new File(dir, "data.temp.zip");
try {
saveToFile(project, tempFile);
} catch (Exception e) {
} catch (IOException e) {
e.printStackTrace();
logger.warn("Failed to save project {}", id);
return;
try {
tempFile.delete();
} catch (Exception e2) {
// just ignore - file probably was never created.
}
throw e;
}
File file = new File(dir, "data.zip");
@ -83,7 +88,7 @@ public class ProjectUtilities {
}
}
protected static void saveToFile(Project project, File file) throws Exception {
protected static void saveToFile(Project project, File file) throws IOException {
ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file));
try {
Pool pool = new Pool();

9
main/src/com/google/refine/model/ColumnModel.java
View File

@ -283,12 +283,17 @@ public class ColumnModel implements Jsonizable {
_nameToColumn = new HashMap<String, Column>();
_cellIndexToColumn = new HashMap<Integer, Column>();
_columnNames = new ArrayList<String>();
int maxCellIndex = -1;
for (Column column : columns) {
_nameToColumn.put(column.getName(), column);
_cellIndexToColumn.put(column.getCellIndex(), column);
int cidx = column.getCellIndex();
if (cidx > maxCellIndex) {
maxCellIndex = cidx;
}
_cellIndexToColumn.put(cidx, column);
_columnNames.add(column.getName());
}
_maxCellIndex = maxCellIndex;
}
/**

5
main/src/com/google/refine/model/Project.java
View File

@ -98,6 +98,9 @@ public class Project {
this.history = new History(this);
}
/**
* Free/dispose of project data from memory.
*/
public void dispose() {
for (OverlayModel overlayModel : overlayModels.values()) {
try {
@ -107,6 +110,7 @@ public class Project {
}
}
ProjectManager.singleton.getInterProjectModel().flushJoinsInvolvingProject(this.id);
// The rest of the project should get garbage collected when we return.
}
public Date getLastSave(){
@ -190,6 +194,7 @@ public class Project {
) throws Exception {
long start = System.currentTimeMillis();
// version of Refine which wrote the file
/* String version = */ reader.readLine();
Project project = new Project(id);

55
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
@ -111,6 +110,9 @@ public class StandardReconConfig extends ReconConfig {
JSONObject t = obj.has("type") && !obj.isNull("type") ? obj.getJSONObject("type") : null;
String limitString = obj.has("limit") && !obj.isNull("limit") ? obj.getString("limit") : "";
int limit = "".equals(limitString) ? 0 : Integer.parseInt(limitString);
return new StandardReconConfig(
obj.getString("service"),
obj.has("identifierSpace") ? obj.getString("identifierSpace") : null,
@ -118,7 +120,8 @@ public class StandardReconConfig extends ReconConfig {
t == null ? null : t.getString("id"),
t == null ? null : (t.has("name") ? t.getString("name") : null),
obj.getBoolean("autoMatch"),
columnDetails
columnDetails,
limit
);
}
@ -140,16 +143,41 @@ public class StandardReconConfig extends ReconConfig {
final public String typeName;
final public boolean autoMatch;
final public List<ColumnDetail> columnDetails;
final private int limit;
public StandardReconConfig(
String service,
String identifierSpace,
String schemaSpace,
String typeID,
String typeName,
boolean autoMatch,
List<ColumnDetail> columnDetails
) {
this(service, identifierSpace, schemaSpace, typeID, typeName, autoMatch, columnDetails, 0);
}
/**
* @param service
* @param identifierSpace
* @param schemaSpace
* @param typeID
* @param typeName
* @param autoMatch
* @param columnDetails
* @param limit maximum number of results to return (0 = default)
*/
public StandardReconConfig(
String service,
String identifierSpace,
String schemaSpace,
String typeID,
String typeName,
boolean autoMatch,
List<ColumnDetail> columnDetails
List<ColumnDetail> columnDetails,
int limit
) {
this.service = service;
this.identifierSpace = identifierSpace;
@ -159,6 +187,7 @@ public class StandardReconConfig extends ReconConfig {
this.typeName = typeName;
this.autoMatch = autoMatch;
this.columnDetails = columnDetails;
this.limit = limit;
}
@Override
@ -190,6 +219,7 @@ public class StandardReconConfig extends ReconConfig {
writer.endObject();
}
writer.endArray();
writer.key("limit"); writer.value(limit);
writer.endObject();
}
@ -266,6 +296,13 @@ public class StandardReconConfig extends ReconConfig {
jsonWriter.endArray();
}
// Only send limit if it's non-default to preserve backward compatibility with
// services which might choke on this
if (limit != 0) {
jsonWriter.key("limit"); jsonWriter.value(limit);
}
jsonWriter.endObject();
job.text = cell.value.toString();
@ -316,11 +353,11 @@ public class StandardReconConfig extends ReconConfig {
}
if (connection.getResponseCode() >= 400) {
// TODO: Retry with backoff on 500 errors?
InputStream is = connection.getErrorStream();
throw new IOException("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
logger.error("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? ""
: ParsingUtilities.inputStreamToString(is));
} else {
InputStream is = connection.getInputStream();
try {
@ -383,7 +420,7 @@ public class StandardReconConfig extends ReconConfig {
try {
int length = results.length();
int count = 0;
for (int i = 0; i < length && count < 3; i++) {
for (int i = 0; i < length; i++) {
JSONObject result = results.getJSONObject(i);
if (!result.has("name")) {
continue;

4
main/src/com/google/refine/util/Pool.java
View File

@ -55,6 +55,10 @@ import com.google.refine.RefineServlet;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
/**
* A serializable pool of ReconCandidates indexed by ID.
*
*/
public class Pool implements Jsonizable {
final protected Map<String, Recon> recons = new HashMap<String, Recon>();

2
main/src/com/google/refine/util/StringUtils.java
View File

@ -16,6 +16,8 @@ public class StringUtils {
if (o instanceof Calendar || o instanceof Date) {
DateFormat formatter = DateFormat.getDateInstance();
return formatter.format(o instanceof Date ? ((Date) o) : ((Calendar) o).getTime());
} else if (o == null) {
return "null";
} else {
return o.toString();
}

82
main/tests/server/src/com/google/refine/tests/HistoryEntryManagerStub.java Normal file
View File

@ -0,0 +1,82 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests;
import java.io.File;
import java.io.Writer;
import java.util.Properties;
import com.google.refine.ProjectManager;
import com.google.refine.history.HistoryEntry;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.io.FileProjectManager;
public class HistoryEntryManagerStub implements HistoryEntryManager{
@Override
public void delete(HistoryEntry historyEntry) {
}
@Override
public void save(HistoryEntry historyEntry, Writer writer, Properties options) {
}
@Override
public void loadChange(HistoryEntry historyEntry) {
}
protected void loadChange(HistoryEntry historyEntry, File file) throws Exception {
}
@Override
public void saveChange(HistoryEntry historyEntry) throws Exception {
}
protected void saveChange(HistoryEntry historyEntry, File file) throws Exception {
}
protected File getChangeFile(HistoryEntry historyEntry) {
return new File(getHistoryDir(historyEntry), historyEntry.id + ".change.zip");
}
protected File getHistoryDir(HistoryEntry historyEntry) {
File dir = new File(((FileProjectManager)ProjectManager.singleton)
.getProjectDir(historyEntry.projectID),
"history");
dir.mkdirs();
return dir;
}
}

3
main/tests/server/src/com/google/refine/tests/ProjectManagerStub.java
View File

@ -62,8 +62,7 @@ public class ProjectManagerStub extends ProjectManager {
@Override
public HistoryEntryManager getHistoryEntryManager() {
// empty
return null;
return new HistoryEntryManagerStub();
}
@Override

178
main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java Normal file
View File

@ -0,0 +1,178 @@
/*
Copyright 2013, Thomas F. Morris
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.expr.functions.booleans;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class BooleanTests extends RefineTest {
private static String TRUTH_TABLE[][] = {
{"and","true","true","true"},
{"and","false","false","false"},
{"and","true","false","false"},
{"and","false","true","false"},
{"or","true","true","true"},
{"or","false","false","false"},
{"or","true","false","true"},
{"or","false","true","true"},
{"xor","true","true","false"},
{"xor","false","false","false"},
{"xor","true","false","true"},
{"xor","false","true","true"},
};
static Properties bindings;
Project project;
Properties options;
JSONObject engine_config;
Engine engine;
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
}
@BeforeMethod
public void SetUp() throws IOException, ModelException {
bindings = new Properties();
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("TNG Test Project");
ProjectManager.singleton.registerProject(project, pm);
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"Column A");
project.columnModel.addColumn(index, column, true);
options = mock(Properties.class);
bindings.put("project", project);
// Five rows of a's and five of 1s
for (int i = 0; i < 10; i++) {
Row row = new Row(1);
row.setCell(0, new Cell(i < 5 ? "a":new Integer(1), null));
project.rows.add(row);
}
}
@AfterMethod
public void TearDown() {
bindings = null;
}
/**
* Lookup a control function by name and invoke it with a variable number of args
*/
private static Object invoke(String name,Object... args) {
// registry uses static initializer, so no need to set it up
Function function = ControlFunctionRegistry.getFunction(name);
if (function == null) {
throw new IllegalArgumentException("Unknown function "+name);
}
if (args == null) {
return function.call(bindings,new Object[0]);
} else {
return function.call(bindings,args);
}
}
@Test
public void testInvalidParams() {
for (String op : new String[] {"and","or","xor"}) {
Assert.assertTrue(invoke(op) instanceof EvalError);
Assert.assertTrue(invoke(op, Boolean.TRUE, Boolean.TRUE, Boolean.TRUE) instanceof EvalError);
Assert.assertTrue(invoke(op, Boolean.TRUE, Integer.valueOf(1)) instanceof EvalError);
Assert.assertTrue(invoke(op, Integer.valueOf(1), Boolean.TRUE) instanceof EvalError);
Assert.assertTrue(invoke(op, Boolean.TRUE,"foo") instanceof EvalError);
Assert.assertTrue(invoke(op, "foo", Boolean.TRUE) instanceof EvalError);
Assert.assertTrue(invoke(op, Boolean.TRUE) instanceof EvalError);
}
String op = "not";
Assert.assertTrue(invoke(op) instanceof EvalError);
Assert.assertTrue(invoke(op, Boolean.TRUE,Boolean.TRUE) instanceof EvalError);
Assert.assertTrue(invoke(op, Integer.valueOf(1)) instanceof EvalError);
Assert.assertTrue(invoke(op, "foo") instanceof EvalError);
}
@Test
public void testBinary() {
for (String[] test : TRUTH_TABLE) {
String operator = test[0];
Boolean op1 = Boolean.valueOf(test[1]);
Boolean op2 = Boolean.valueOf(test[2]);
Boolean result = Boolean.valueOf(test[3]);
Assert.assertEquals(invoke(operator, op1, op2),result);
}
Assert.assertEquals(invoke("not", Boolean.TRUE),Boolean.FALSE);
Assert.assertEquals(invoke("not", Boolean.FALSE),Boolean.TRUE);
}
}

21
main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java
View File

@ -103,6 +103,7 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertNull(invoke("toNumber", (Object) null));
Assert.assertTrue(invoke("toNumber", "string") instanceof EvalError);
Assert.assertEquals(invoke("toNumber", "0.0"), 0.0);
Assert.assertEquals(invoke("toNumber", "123"), Long.valueOf(123));
Assert.assertTrue(Math.abs((Double) invoke("toNumber", "123.456") - 123.456) < EPSILON);
Assert.assertTrue(Math.abs((Double) invoke("toNumber", "001.234") - 1.234) < EPSILON);
Assert.assertTrue(Math.abs((Double) invoke("toNumber", "1e2") - 100.0) < EPSILON);
@ -110,13 +111,16 @@ public class ToFromConversionTests extends RefineTest {
}
@Test
public void testToString() {
public void testToString() throws CalendarParserException {
Assert.assertTrue(invoke("toString") instanceof EvalError);
Assert.assertTrue(invoke("toString", (Object) null) instanceof EvalError);
Assert.assertEquals(invoke("toString", (Object) null), "null");
Assert.assertEquals(invoke("toString", Long.valueOf(100)),"100");
Assert.assertEquals(invoke("toString", Double.valueOf(100.0)),"100.0");
// Calendar
// Date
// Date&Calendar with 2nd parameter with format
Assert.assertEquals(invoke("toString", Double.valueOf(100.0),"%.0f"),"100");
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01")),"Jun 1, 2013");
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01").getTime()),"Jun 1, 2013");
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01"),"yyyy"),"2013");
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01"),"yyyy-MM-dd"),"2013-06-01");
}
@Test
@ -131,6 +135,13 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parse("2012-03-01"));
// Multiple format strings should get tried sequentially until one succeeds or all are exhausted
Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parse("2012-03-01"));
// First string can be a locale identifier instead of a format string
Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_HK","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_TW","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_CN","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
// Date
// Calendar
// String

2
main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
View File

@ -183,7 +183,7 @@ public class ExcelImporterTests extends ImporterTest {
File file = null;
try {
file = File.createTempFile("oepnrefine-importer-test", xml ? ".xlsx" : ".xls");
file = File.createTempFile("openrefine-importer-test", xml ? ".xlsx" : ".xls");
file.deleteOnExit();
OutputStream outputStream = new FileOutputStream(file);
wb.write(outputStream);

43
main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java
View File

@ -84,7 +84,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3";
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -104,7 +104,7 @@ public class TsvCsvImporterTests extends ImporterTest {
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -129,7 +129,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "234" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 0, 0, 1, true, true, false);
prepareOptions(sep, -1, 0, 0, 1, true, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -153,7 +153,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 0, 0, 0, false, true, false);
prepareOptions(sep, -1, 0, 0, 0, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -176,7 +176,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = " data1 " + inputSeparator + " 3.4 " + inputSeparator + " data3 ";
try {
prepareOptions(sep, -1, 0, 0, 0, false, true, false);
prepareOptions(sep, -1, 0, 0, 0, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -196,7 +196,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = " data1" + inputSeparator + " 12" + inputSeparator + " data3";
try {
prepareOptions(sep, -1, 0, 0, 0, true, true, false);
prepareOptions(sep, -1, 0, 0, 0, true, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -216,7 +216,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = " data1" + inputSeparator + inputSeparator + " data3";
try {
prepareOptions(sep, -1, 0, 0, 0, true, true, false);
prepareOptions(sep, -1, 0, 0, 0, true, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -238,7 +238,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "data2" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 0, 0, 2, false, true, false);
prepareOptions(sep, -1, 0, 0, 2, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -262,7 +262,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "data2" + inputSeparator + "data3" + inputSeparator + "data4" + inputSeparator + "data5" + inputSeparator + "data6";
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -292,7 +292,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\"" + inputSeparator + "data2";
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -316,7 +316,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "data2" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 0, 1, 1, false, true, false);
prepareOptions(sep, -1, 0, 1, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -341,7 +341,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "data2" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 1, 0, 1, false, true, false);
prepareOptions(sep, -1, 1, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -370,7 +370,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data1" + inputSeparator + "data2" + inputSeparator + "data3";
try {
prepareOptions(sep, -1, 1, 3, 2, false, true, false);
prepareOptions(sep, -1, 1, 3, 2, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -402,7 +402,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"data-row3-cell1" + inputSeparator + "data-row3-cell2" + inputSeparator + "data-row1-cell3";
try {
prepareOptions(sep, 2, 2, 3, 2, false, true, false);
prepareOptions(sep, 2, 2, 3, 2, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -428,7 +428,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String inputSeparator = sep == null ? "\t" : sep;
String input = "data1" + inputSeparator + "data2\"" + inputSeparator + "data3" + inputSeparator + "data4";
try {
prepareOptions(sep, -1, 0, 0, 0, false, true, true);
prepareOptions(sep, -1, 0, 0, 0, false, true);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -449,7 +449,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\"" + inputSeparator + "data2";
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -472,7 +472,7 @@ public class TsvCsvImporterTests extends ImporterTest {
"\"A line with many \n\n\n\n\n empty lines\"" + inputSeparator + "data2";
try {
prepareOptions(sep, -1, 0, 0, 1, false, true, false);
prepareOptions(sep, -1, 0, 0, 1, false, false);
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
@ -492,7 +492,7 @@ public class TsvCsvImporterTests extends ImporterTest {
public void readCsvWithProperties() {
StringReader reader = new StringReader(SAMPLE_ROW);
prepareOptions(",", -1, 0, 0, 0, true, true, true);
prepareOptions(",", -1, 0, 0, 0, true, true);
try {
parseOneFile(SUT, reader);
@ -514,7 +514,7 @@ public class TsvCsvImporterTests extends ImporterTest {
String input = "data1,data2\",data3,data4";
StringReader reader = new StringReader(input);
prepareOptions(",", -1, 0, 0, 0, true, true, true);
prepareOptions(",", -1, 0, 0, 0, true, true);
try {
parseOneFile(SUT, reader);
@ -545,7 +545,7 @@ public class TsvCsvImporterTests extends ImporterTest {
private void prepareOptions(
String sep, int limit, int skip, int ignoreLines,
int headerLines, boolean guessValueType, boolean splitIntoColumns, boolean ignoreQuotes) {
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
whenGetStringOption("separator", options, sep);
whenGetIntegerOption("limit", options, limit);
@ -553,7 +553,6 @@ public class TsvCsvImporterTests extends ImporterTest {
whenGetIntegerOption("ignoreLines", options, ignoreLines);
whenGetIntegerOption("headerLines", options, headerLines);
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
whenGetBooleanOption("splitIntoColumns", options, splitIntoColumns);
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
}
@ -568,8 +567,6 @@ public class TsvCsvImporterTests extends ImporterTest {
verify(options, times(1)).getBoolean("guessCellValueTypes");
verify(options, times(1)).getBoolean("processQuotes");
verify(options, times(1)).getBoolean("storeBlankCellsAsNulls");
// TODO: Is this option not being read? - tfm
// verify(options, times(1)).getBoolean("splitIntoColumns");
} catch (JSONException e) {
Assert.fail("JSON exception",e);
}

BIN
main/tests/server/src/com/google/refine/tests/importers/excel95.xls Normal file
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar Normal file
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib-src/butterfly-trunk-sources.jar
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib-src/opencsv-2.4-SNAPSHOT-sources.jar
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/butterfly-1.0.1.jar Normal file
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/butterfly-trunk.jar
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More