Merge pull request #1301 from jackyq2015/master

metadata for project
This commit is contained in:
Jacky 2017-11-04 17:35:13 -04:00 committed by GitHub
commit 8d84acb391
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 3710 additions and 73 deletions

View File

@ -17,7 +17,6 @@
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
@ -77,7 +76,6 @@
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jackson-core-2.1.3.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.13-20150929.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/>
@ -85,6 +83,9 @@
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.7.1.jar"/>
<classpathentry kind="lib" path="main/tests/data"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/swc-parser-lazy-3.1.5-jar-with-dependencies.jar" sourcepath="main/webapp/WEB-INF/lib-src/swc-parser-lazy-3.1.5-sources.jar" />
<classpathentry kind="output" path="build"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/swc-parser-lazy-3.1.5-jar-with-dependencies.jar" sourcepath="main/webapp/WEB-INF/lib-src/swc-parser-lazy-3.1.5-sources.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-annotations-2.9.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-2.9.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-databind-2.9.1.jar"/>
<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
</classpath>

28
.gitignore vendored
View File

@ -1,28 +0,0 @@
*~
\#*#
.*.swp
*.DS_Store
*.class
.com.apple.timemachine.supported
.import-temp/
build/
dist/
server/classes/
main/webapp/WEB-INF/classes/
main/tests/server/classes/
main/test-output/
appengine/classes/
tools/
extensions/sample-extension/module/MOD-INF/classes/
extensions/jython/module/MOD-INF/classes/
extensions/jython/module/MOD-INF/lib/cachedir/
extensions/rdf-exporter/module/MOD-INF/classes/
broker/appengine/module/MOD-INF/classes/
broker/core/module/MOD-INF/classes/
broker/core/WEB-INF/lib/
broker/core/data/
broker/core/test-output/
tmp/
/test-output
/bin
open-refine.log

View File

@ -88,5 +88,7 @@ public class PCAxisImporter extends TabularImportingParserBase {
TabularImportingParserBase.readTable(
project, metadata, job, dataReader,
fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
}

View File

@ -193,7 +193,7 @@ public abstract class ProjectManager {
* @param projectId
* @throws Exception
*/
protected abstract void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception;
public abstract void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception;
/**
* Save project to the data store

View File

@ -34,12 +34,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
@ -55,16 +58,28 @@ public class ProjectMetadata implements Jsonizable {
private final Date _created;
private Date _modified;
private Date written = null;
private String _name;
private String _password;
private String _name = "";
private String _password = "";
private String _encoding;
private String _encoding = "";
private int _encodingConfidence;
private String _creator = "";
private String _contributors = "";
private String _subject = ""; // Several refine projects may be linked
private String _description = ""; // free form of comment
private int _rowNumber; // at the creation. Essential for cleaning old projects too heavy
// import options is an array for 1-n data sources
private JSONArray _importOptionMetaData = new JSONArray();
// user metadata
private JSONArray _userMetadata = new JSONArray();;
private Map<String, Serializable> _customMetadata = new HashMap<String, Serializable>();
private PreferenceStore _preferenceStore = new PreferenceStore();
final Logger logger = LoggerFactory.getLogger("project_metadata");
private final static Logger logger = LoggerFactory.getLogger("project_metadata");
protected ProjectMetadata(Date date) {
_created = date;
@ -90,7 +105,12 @@ public class ProjectMetadata implements Jsonizable {
writer.key("name"); writer.value(_name);
writer.key("created"); writer.value(ParsingUtilities.dateToString(_created));
writer.key("modified"); writer.value(ParsingUtilities.dateToString(_modified));
writer.key("creator"); writer.value(_creator);
writer.key("contributors"); writer.value(_contributors);
writer.key("subject"); writer.value(_subject);
writer.key("description"); writer.value(_description);
writer.key("rowNumber"); writer.value(_rowNumber);
writer.key("customMetadata"); writer.object();
for (String key : _customMetadata.keySet()) {
Serializable value = _customMetadata.get(key);
@ -99,7 +119,19 @@ public class ProjectMetadata implements Jsonizable {
}
writer.endObject();
if ("save".equals(options.getProperty("mode"))) {
// write JSONArray directly
if (_importOptionMetaData.length() > 0 ) {
writer.key("importOptionMetaData");
writer.value(_importOptionMetaData);
}
// write user metadata in {name, value, display} form:
if (_userMetadata.length() > 0) {
writer.key(PreferenceStore.USER_METADATA_KEY);
writer.value(_userMetadata);
}
if (isSaveMode(options)) {
writer.key("password"); writer.value(_password);
writer.key("encoding"); writer.value(_encoding);
@ -110,10 +142,20 @@ public class ProjectMetadata implements Jsonizable {
writer.endObject();
if ("save".equals(options.getProperty("mode"))) {
if (isSaveMode(options)) {
written = new Date();
}
}
public void writeWithoutOption(JSONWriter writer)
throws JSONException {
write(writer,
new Properties());
}
private boolean isSaveMode(Properties options) {
return "save".equals(options.getProperty("mode"));
}
public boolean isDirty() {
return written == null || _modified.after(written);
@ -147,12 +189,18 @@ public class ProjectMetadata implements Jsonizable {
pm._encoding = JSONUtilities.getString(obj, "encoding", "");
pm._encodingConfidence = JSONUtilities.getInt(obj, "encodingConfidence", 0);
pm._creator = JSONUtilities.getString(obj, "creator", "");
pm._contributors = JSONUtilities.getString(obj, "contributors", "");
pm._subject = JSONUtilities.getString(obj, "subject", "");
pm._description = JSONUtilities.getString(obj, "description", "");
pm._rowNumber = JSONUtilities.getInt(obj, "rowNumber", 0);
if (obj.has("preferences") && !obj.isNull("preferences")) {
try {
pm._preferenceStore.load(obj.getJSONObject("preferences"));
} catch (JSONException e) {
// ignore
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
@ -161,7 +209,7 @@ public class ProjectMetadata implements Jsonizable {
((TopList) pm._preferenceStore.get("scripting.expressions"))
.load(obj.getJSONArray("expressions"));
} catch (JSONException e) {
// ignore
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
@ -179,10 +227,28 @@ public class ProjectMetadata implements Jsonizable {
}
}
} catch (JSONException e) {
// ignore
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
if (obj.has("importOptionMetaData") && !obj.isNull("importOptionMetaData")) {
try {
JSONArray jsonArray = obj.getJSONArray("importOptionMetaData");
pm._importOptionMetaData = jsonArray;
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
if (obj.has(PreferenceStore.USER_METADATA_KEY) && !obj.isNull(PreferenceStore.USER_METADATA_KEY)) {
try {
JSONArray jsonArray = obj.getJSONArray(PreferenceStore.USER_METADATA_KEY);
pm._userMetadata = jsonArray;
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
pm.written = new Date(); // Mark it as not needing writing until modified
return pm;
@ -263,4 +329,110 @@ public class ProjectMetadata implements Jsonizable {
}
updateModified();
}
public JSONArray getImportOptionMetaData() {
return _importOptionMetaData;
}
public void setImportOptionMetaData(JSONArray jsonArray) {
_importOptionMetaData = jsonArray;
updateModified();
}
public void appendImportOptionMetaData(JSONObject obj) {
_importOptionMetaData.put(obj);
updateModified();
}
public String getCreator() {
return _creator;
}
public void setCreator(String creator) {
this._creator = creator;
updateModified();
}
public String getContributors() {
return _contributors;
}
public void setContributors(String contributors) {
this._contributors = contributors;
updateModified();
}
public String getSubject() {
return _subject;
}
public void setSubject(String subject) {
this._subject = subject;
updateModified();
}
public String getDescription() {
return _description;
}
public void setDescription(String description) {
this._description = description;
updateModified();
}
public int getRowNumber() {
return _rowNumber;
}
public void setRowNumber(int rowNumber) {
this._rowNumber = rowNumber;
updateModified();
}
public JSONArray getUserMetadata() {
return _userMetadata;
}
public void setUserMetadata(JSONArray userMetadata) {
this._userMetadata = userMetadata;
}
private void updateUserMetadata(String metaName, String valueString) {
for (int i = 0; i < _userMetadata.length(); i++) {
try {
JSONObject obj = _userMetadata.getJSONObject(i);
if (obj.getString("name").equals(metaName)) {
obj.put("value", valueString);
}
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
}
public void setAnyField(String metaName, String valueString) {
Class<? extends ProjectMetadata> metaClass = this.getClass();
try {
Field metaField = metaClass.getDeclaredField("_" + metaName);
metaField.set(this, valueString);
} catch (NoSuchFieldException e) {
updateUserMetadata(metaName, valueString);
} catch (SecurityException | IllegalArgumentException | IllegalAccessException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
}

View File

@ -0,0 +1,47 @@
package com.google.refine.commands.project;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
public class SetProjectMetaDataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Project project = request.getParameter("project") != null ? getProject(request) : null;
String metaName = request.getParameter("name");
String valueString = request.getParameter("value");
ProjectMetadata meta = null;
if (project == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"Project cannot be found\" }");
return;
}
meta = project.getMetadata();
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
meta.setAnyField(metaName, valueString);
ProjectManager.singleton.saveMetadata(meta, project.id);
respond(response, "{ \"code\" : \"ok\" }");
} catch (JSONException e) {
respondException(response, e);
} catch (Exception e) {
respondException(response, e);
}
}
}

View File

@ -42,6 +42,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONWriter;
@ -62,6 +63,7 @@ public class GetAllProjectMetadataCommand extends Command {
Properties options = new Properties();
writer.object();
writer.key("projects");
writer.object();
Map<Long, ProjectMetadata> m = ProjectManager.singleton.getAllProjectMetadata();
@ -73,6 +75,12 @@ public class GetAllProjectMetadataCommand extends Command {
}
}
writer.endObject();
writer.key("customMetaDataColumns");
JSONArray customMetaDataColumns = new JSONArray(
(String)ProjectManager.singleton.getPreferenceStore().get("userMetaData"));
writer.value(customMetaDataColumns);
writer.endObject();
} catch (JSONException e) {
respondException(response, e);

View File

@ -225,6 +225,8 @@ public class ExcelImporter extends TabularImportingParserBase {
exceptions
);
}
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
static protected Serializable extractCell(org.apache.poi.ss.usermodel.Cell cell) {

View File

@ -107,6 +107,8 @@ public class FixedWidthImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
/**

View File

@ -39,7 +39,7 @@ import java.io.InputStream;
import java.io.Reader;
import java.util.List;
import org.apache.commons.lang.NotImplementedException;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -145,7 +145,17 @@ abstract public class ImportingParserBase implements ImportingParser {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
pushImportingOptions(metadata, fileSource, options);
}
private void pushImportingOptions(ProjectMetadata metadata, String fileSource, JSONObject options) {
try {
options.put("fileSource", fileSource);
} catch (JSONException e) {
// ignore
}
// set the import options to metadata:
metadata.appendImportOptionMetaData(options);
}
public void parseOneFile(
@ -158,7 +168,7 @@ abstract public class ImportingParserBase implements ImportingParser {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
pushImportingOptions(metadata, fileSource, options);
}

View File

@ -199,6 +199,8 @@ public class JsonImporter extends TreeImportingParserBase {
parseOneFile(project, metadata, job, fileSource,
new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, is, rootColumnGroup, limit, options, exceptions);
}
static public class JSONTreeReader implements TreeReader {

View File

@ -104,5 +104,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
}

View File

@ -183,6 +183,8 @@ public class OdsImporter extends TabularImportingParserBase {
exceptions
);
}
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
static protected Serializable extractCell(OdfTableCell cell) {

View File

@ -160,5 +160,7 @@ public class RdfTripleImporter extends ImportingParserBase {
} finally {
triples.iterator().close();
}
super.parseOneFile(project, metadata, job, fileSource, input, limit, options, exceptions);
}
}

View File

@ -121,6 +121,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, lnReader, limit, options, exceptions);
}
static protected ArrayList<Object> getCells(String line, CSVParser parser, LineNumberReader lnReader)

View File

@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@ -200,4 +201,9 @@ abstract public class TabularImportingParserBase extends ImportingParserBase {
exceptions.add(e);
}
}
public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource,
Reader dataReader, int limit, JSONObject options, List<Exception> exceptions) {
super.parseOneFile(project, metadata, job, fileSource, dataReader, limit, options, exceptions);
}
}

View File

@ -731,6 +731,8 @@ public class WikitextImporter extends TabularImportingParserBase {
exceptions.add(e1);
e1.printStackTrace();
}
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
private StandardReconConfig getReconConfig(String url) {

View File

@ -201,6 +201,8 @@ public class XmlImporter extends TreeImportingParserBase {
try {
parseOneFile(project, metadata, job, fileSource,
new XmlParser(inputStream), rootColumnGroup, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, inputStream, rootColumnGroup, limit, options, exceptions);
} catch (XMLStreamException e) {
exceptions.add(e);
} catch (IOException e) {

View File

@ -2,11 +2,11 @@ package com.google.refine.importers.tree;
public class ImportParameters {
boolean trimStrings;
boolean storeEmptyStrings;
boolean guessDataType;
boolean includeFileSources;
String fileSource;
protected boolean trimStrings;
protected boolean storeEmptyStrings;
protected boolean guessDataType;
protected boolean includeFileSources;
protected String fileSource;
public ImportParameters(boolean trimStrings, boolean storeEmptyStrings, boolean guessCellValueTypes,
boolean includeFileSources, String fileSource) {

View File

@ -174,7 +174,8 @@ abstract public class TreeImportingParserBase extends ImportingParserBase {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
// throw new NotImplementedException();
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
/**

View File

@ -419,7 +419,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
boolean includeFileSources, String fileSource) {
for (List<Cell> row : record.rows) {
if (row.size() > 0) {
Row realRow = new Row(row.size()); ;
Row realRow = new Row(row.size());
for (int c = 0; c < row.size(); c++) {
if (c == 0 && includeFileSources) { // to add the file source:
realRow.setCell(

View File

@ -44,6 +44,7 @@ import java.io.OutputStream;
import java.util.Properties;
import java.util.zip.GZIPInputStream;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.apache.tools.tar.TarOutputStream;
@ -59,8 +60,10 @@ import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.preference.TopList;
public class FileProjectManager extends ProjectManager {
final static protected String PROJECT_DIR_SUFFIX = ".project";
@ -215,7 +218,7 @@ public class FileProjectManager extends ProjectManager {
}
@Override
protected void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception {
public void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception {
File projectDir = getProjectDir(projectId);
ProjectMetadataUtilities.save(metadata, projectDir);
}
@ -283,6 +286,8 @@ public class FileProjectManager extends ProjectManager {
if (metadata != null) {
jsonWriter.value(id);
if (metadata.isDirty()) {
Project project = ProjectManager.singleton.getProject(id);
metadata.setRowNumber(project.rows.size());
ProjectMetadataUtilities.save(metadata, getProjectDir(id));
saveWasNeeded = true;
}
@ -355,6 +360,11 @@ public class FileProjectManager extends ProjectManager {
reader = new FileReader(file);
JSONTokener tokener = new JSONTokener(reader);
JSONObject obj = (JSONObject) tokener.nextValue();
// load global preferences firstly
if (obj.has("preferences") && !obj.isNull("preferences")) {
_preferenceStore.load(obj.getJSONObject("preferences"));
}
JSONArray a = obj.getJSONArray("projectIDs");
int count = a.length();
@ -363,14 +373,12 @@ public class FileProjectManager extends ProjectManager {
File projectDir = getProjectDir(id);
ProjectMetadata metadata = ProjectMetadataUtilities.load(projectDir);
mergeEmptyUserMetadata(metadata);
_projectsMetadata.put(id, metadata);
}
if (obj.has("preferences") && !obj.isNull("preferences")) {
_preferenceStore.load(obj.getJSONObject("preferences"));
}
if (obj.has("expressions") && !obj.isNull("expressions")) { // backward compatibility
((TopList) _preferenceStore.get("scripting.expressions"))
.load(obj.getJSONArray("expressions"));
@ -395,6 +403,53 @@ public class FileProjectManager extends ProjectManager {
return found;
}
private void mergeEmptyUserMetadata(ProjectMetadata metadata) {
if (metadata == null)
return;
// place holder
JSONArray userMetadataPreference = null;
// actual metadata for project
JSONArray jsonObjArray = metadata.getUserMetadata();
try {
userMetadataPreference = new JSONArray((String)_preferenceStore.get(PreferenceStore.USER_METADATA_KEY));
} catch (JSONException e1) {
logger.error(ExceptionUtils.getFullStackTrace(e1));
}
if (userMetadataPreference == null) {
logger.warn("wrong definition of userMetadata format. Please use form [{\"name\": \"client name\", \"display\":true}, {\"name\": \"progress\", \"display\":false}]");
return;
}
for (int index = 0; index < userMetadataPreference.length(); index++) {
try {
boolean found = false;
JSONObject placeHolderJsonObj = userMetadataPreference.getJSONObject(index);
for (int i = 0; i < jsonObjArray.length(); i++) {
JSONObject jsonObj = jsonObjArray.getJSONObject(i);
if (jsonObj.getString("name").equals(placeHolderJsonObj.getString("name"))) {
found = true;
jsonObj.put("display", placeHolderJsonObj.get("display"));
break;
}
}
if (!found) {
placeHolderJsonObj.put("value", "");
metadata.getUserMetadata().put(placeHolderJsonObj);
logger.info("Put the placeholder {} for project {}",
placeHolderJsonObj.getString("name"),
metadata.getName());
}
} catch (JSONException e) {
logger.warn("Exception when mergeEmptyUserMetadata",e);
}
}
}
protected void recover() {
boolean recovered = false;
for (File file : _workspaceDir.listFiles()) {

View File

@ -48,6 +48,8 @@ import com.google.refine.Jsonizable;
import com.google.refine.RefineServlet;
public class PreferenceStore implements Jsonizable {
public static final String USER_METADATA_KEY = "userMetaData";
private boolean dirty = false;
protected Map<String, Object> _prefs = new HashMap<String, Object>();

View File

@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.model;
package com.google.refine.tests.operations.cell;
import static org.mockito.Mockito.mock;

View File

@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.model;
package com.google.refine.tests.operations.cell;
import static org.mockito.Mockito.mock;

View File

@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.operations.cell;
package com.google.refine.tests.operations.cell;
import static org.mockito.Mockito.mock;
@ -57,6 +57,7 @@ import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.cell.KeyValueColumnizeOperation;
import com.google.refine.process.Process;
import com.google.refine.tests.ProjectManagerStub;
import com.google.refine.tests.RefineServletStub;

View File

@ -68,9 +68,11 @@ function registerCommands() {
RS.registerCommand(module, "get-project-metadata", new Packages.com.google.refine.commands.project.GetProjectMetadataCommand());
RS.registerCommand(module, "get-all-project-metadata", new Packages.com.google.refine.commands.workspace.GetAllProjectMetadataCommand());
RS.registerCommand(module, "set-metaData", new Packages.com.google.refine.commands.project.SetProjectMetaDataCommand());
RS.registerCommand(module, "delete-project", new Packages.com.google.refine.commands.project.DeleteProjectCommand());
RS.registerCommand(module, "rename-project", new Packages.com.google.refine.commands.project.RenameProjectCommand());
RS.registerCommand(module, "set-project-metadata", new Packages.com.google.refine.commands.project.SetProjectMetaDataCommand());
RS.registerCommand(module, "get-models", new Packages.com.google.refine.commands.project.GetModelsCommand());
RS.registerCommand(module, "get-rows", new Packages.com.google.refine.commands.row.GetRowsCommand());
@ -317,6 +319,7 @@ function init() {
"externals/jquery-ui/jquery-ui-1.10.3.custom.js",
"externals/date.js",
"externals/jquery.i18n.js",
"externals/tablesorter/jquery.tablesorter.min.js",
"scripts/util/misc.js",
"scripts/util/url.js",
@ -350,7 +353,8 @@ function init() {
"scripts/index/parser-interfaces/rdf-triples-parser-ui.js",
"scripts/index/parser-interfaces/wikitext-parser-ui.js",
"scripts/reconciliation/recon-manager.js" // so that reconciliation functions are available to importers
"scripts/reconciliation/recon-manager.js", // so that reconciliation functions are available to importers
"scripts/index/edit-metadata-dialog.js"
]
);
@ -359,6 +363,7 @@ function init() {
module,
[
"externals/jquery-ui/css/ui-lightness/jquery-ui-1.10.3.custom.css",
"externals/tablesorter/theme.blue.css",
"styles/jquery-ui-overrides.less",
"styles/common.less",
"styles/pure.css",

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,229 @@
/*************
Blue Theme
*************/
/* overall */
.tablesorter-blue {
width: 100%;
background-color: #fff;
margin: 10px 0 15px;
text-align: left;
border-spacing: 0;
border: #cdcdcd 1px solid;
border-width: 1px 0 0 1px;
}
.tablesorter-blue th,
.tablesorter-blue td {
border: #cdcdcd 1px solid;
border-width: 0 1px 1px 0;
}
/* header */
.tablesorter-blue th,
.tablesorter-blue thead td {
font: 12px/18px Arial, Sans-serif;
font-weight: bold;
color: #000;
background-color: #99bfe6;
border-collapse: collapse;
padding: 4px;
text-shadow: 0 1px 0 rgba(204, 204, 204, 0.7);
}
.tablesorter-blue tbody td,
.tablesorter-blue tfoot th,
.tablesorter-blue tfoot td {
padding: 4px;
vertical-align: top;
}
.tablesorter-blue .header,
.tablesorter-blue .tablesorter-header {
/* black (unsorted) double arrow */
background-image: url(data:image/gif;base64,R0lGODlhFQAJAIAAACMtMP///yH5BAEAAAEALAAAAAAVAAkAAAIXjI+AywnaYnhUMoqt3gZXPmVg94yJVQAAOw==);
/* white (unsorted) double arrow */
/* background-image: url(data:image/gif;base64,R0lGODlhFQAJAIAAAP///////yH5BAEAAAEALAAAAAAVAAkAAAIXjI+AywnaYnhUMoqt3gZXPmVg94yJVQAAOw==); */
/* image */
/* background-image: url(images/black-unsorted.gif); */
background-repeat: no-repeat;
background-position: center right;
padding: 4px 18px 4px 4px;
white-space: normal;
cursor: pointer;
}
.tablesorter-blue .headerSortUp,
.tablesorter-blue .tablesorter-headerSortUp,
.tablesorter-blue .tablesorter-headerAsc {
background-color: #9fbfdf;
/* black asc arrow */
background-image: url(data:image/gif;base64,R0lGODlhFQAEAIAAACMtMP///yH5BAEAAAEALAAAAAAVAAQAAAINjI8Bya2wnINUMopZAQA7);
/* white asc arrow */
/* background-image: url(data:image/gif;base64,R0lGODlhFQAEAIAAAP///////yH5BAEAAAEALAAAAAAVAAQAAAINjI8Bya2wnINUMopZAQA7); */
/* image */
/* background-image: url(images/black-asc.gif); */
}
.tablesorter-blue .headerSortDown,
.tablesorter-blue .tablesorter-headerSortDown,
.tablesorter-blue .tablesorter-headerDesc {
background-color: #8cb3d9;
/* black desc arrow */
background-image: url(data:image/gif;base64,R0lGODlhFQAEAIAAACMtMP///yH5BAEAAAEALAAAAAAVAAQAAAINjB+gC+jP2ptn0WskLQA7);
/* white desc arrow */
/* background-image: url(data:image/gif;base64,R0lGODlhFQAEAIAAAP///////yH5BAEAAAEALAAAAAAVAAQAAAINjB+gC+jP2ptn0WskLQA7); */
/* image */
/* background-image: url(images/black-desc.gif); */
}
.tablesorter-blue thead .sorter-false {
background-image: none;
cursor: default;
padding: 4px;
}
/* tfoot */
.tablesorter-blue tfoot .tablesorter-headerSortUp,
.tablesorter-blue tfoot .tablesorter-headerSortDown,
.tablesorter-blue tfoot .tablesorter-headerAsc,
.tablesorter-blue tfoot .tablesorter-headerDesc {
/* remove sort arrows from footer */
background-image: none;
}
/* tbody */
.tablesorter-blue td {
color: #3d3d3d;
background-color: #fff;
padding: 4px;
vertical-align: top;
}
/* hovered row colors
you'll need to add additional lines for
rows with more than 2 child rows
*/
.tablesorter-blue tbody > tr.hover > td,
.tablesorter-blue tbody > tr:hover > td,
.tablesorter-blue tbody > tr:hover + tr.tablesorter-childRow > td,
.tablesorter-blue tbody > tr:hover + tr.tablesorter-childRow + tr.tablesorter-childRow > td,
.tablesorter-blue tbody > tr.even.hover > td,
.tablesorter-blue tbody > tr.even:hover > td,
.tablesorter-blue tbody > tr.even:hover + tr.tablesorter-childRow > td,
.tablesorter-blue tbody > tr.even:hover + tr.tablesorter-childRow + tr.tablesorter-childRow > td {
background-color: #d9d9d9;
}
.tablesorter-blue tbody > tr.odd.hover > td,
.tablesorter-blue tbody > tr.odd:hover > td,
.tablesorter-blue tbody > tr.odd:hover + tr.tablesorter-childRow > td,
.tablesorter-blue tbody > tr.odd:hover + tr.tablesorter-childRow + tr.tablesorter-childRow > td {
background-color: #bfbfbf;
}
/* table processing indicator */
.tablesorter-blue .tablesorter-processing {
background-position: center center !important;
background-repeat: no-repeat !important;
/* background-image: url(images/loading.gif) !important; */
background-image: url('data:image/gif;base64,R0lGODlhFAAUAKEAAO7u7lpaWgAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQBCgACACwAAAAAFAAUAAACQZRvoIDtu1wLQUAlqKTVxqwhXIiBnDg6Y4eyx4lKW5XK7wrLeK3vbq8J2W4T4e1nMhpWrZCTt3xKZ8kgsggdJmUFACH5BAEKAAIALAcAAAALAAcAAAIUVB6ii7jajgCAuUmtovxtXnmdUAAAIfkEAQoAAgAsDQACAAcACwAAAhRUIpmHy/3gUVQAQO9NetuugCFWAAAh+QQBCgACACwNAAcABwALAAACE5QVcZjKbVo6ck2AF95m5/6BSwEAIfkEAQoAAgAsBwANAAsABwAAAhOUH3kr6QaAcSrGWe1VQl+mMUIBACH5BAEKAAIALAIADQALAAcAAAIUlICmh7ncTAgqijkruDiv7n2YUAAAIfkEAQoAAgAsAAAHAAcACwAAAhQUIGmHyedehIoqFXLKfPOAaZdWAAAh+QQFCgACACwAAAIABwALAAACFJQFcJiXb15zLYRl7cla8OtlGGgUADs=') !important;
}
/* Zebra Widget - row alternating colors */
.tablesorter-blue tbody tr.odd > td {
background-color: #ebf2fa;
}
.tablesorter-blue tbody tr.even > td {
background-color: #fff;
}
/* Column Widget - column sort colors */
.tablesorter-blue td.primary,
.tablesorter-blue tr.odd td.primary {
background-color: #99b3e6;
}
.tablesorter-blue tr.even td.primary {
background-color: #c2d1f0;
}
.tablesorter-blue td.secondary,
.tablesorter-blue tr.odd td.secondary {
background-color: #c2d1f0;
}
.tablesorter-blue tr.even td.secondary {
background-color: #d6e0f5;
}
.tablesorter-blue td.tertiary,
.tablesorter-blue tr.odd td.tertiary {
background-color: #d6e0f5;
}
.tablesorter-blue tr.even td.tertiary {
background-color: #ebf0fa;
}
/* caption */
caption {
background-color: #fff;
}
/* filter widget */
.tablesorter-blue .tablesorter-filter-row {
background-color: #eee;
}
.tablesorter-blue .tablesorter-filter-row td {
background-color: #eee;
line-height: normal;
text-align: center; /* center the input */
-webkit-transition: line-height 0.1s ease;
-moz-transition: line-height 0.1s ease;
-o-transition: line-height 0.1s ease;
transition: line-height 0.1s ease;
}
/* optional disabled input styling */
.tablesorter-blue .tablesorter-filter-row .disabled {
opacity: 0.5;
filter: alpha(opacity=50);
cursor: not-allowed;
}
/* hidden filter row */
.tablesorter-blue .tablesorter-filter-row.hideme td {
/*** *********************************************** ***/
/*** change this padding to modify the thickness ***/
/*** of the closed filter row (height = padding x 2) ***/
padding: 2px;
/*** *********************************************** ***/
margin: 0;
line-height: 0;
cursor: pointer;
}
.tablesorter-blue .tablesorter-filter-row.hideme * {
height: 1px;
min-height: 0;
border: 0;
padding: 0;
margin: 0;
/* don't use visibility: hidden because it disables tabbing */
opacity: 0;
filter: alpha(opacity=0);
}
/* filters */
.tablesorter-blue input.tablesorter-filter,
.tablesorter-blue select.tablesorter-filter {
width: 98%;
height: auto;
margin: 0;
padding: 4px;
background-color: #fff;
border: 1px solid #bbb;
color: #333;
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
-webkit-transition: height 0.1s ease;
-moz-transition: height 0.1s ease;
-o-transition: height 0.1s ease;
transition: height 0.1s ease;
}
/* rows hidden by filtering (needed for child rows) */
.tablesorter .filtered {
display: none;
}
/* ajax error row */
.tablesorter .tablesorter-errorRow td {
text-align: center;
cursor: pointer;
background-color: #e6bf99;
}

View File

@ -58,6 +58,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
<div id="openrefine-version"></div>
</div>
<ul>
<li><a href="preferences" id="or-index-pref"></a></li>
<li><a href="https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users" id="or-index-help"></a></li>
<li><a href="about.html" id="or-index-about"></a></li>
</ul>

View File

@ -21,7 +21,8 @@
"error-rename": "Failed to rename project:",
"no-proj": "No existing project. Select 'Create Project' on the left to create a new project.",
"try-these": "If you have no data to work with, try these",
"sample-data": "sample data sets"
"sample-data": "sample data sets",
"change-metadata-value": "Change value of metadata key"
},
"core-index-create": {
"create-proj": "Create Project",
@ -87,6 +88,12 @@
"open-proj": "Open Project",
"name": "Name",
"rename": "rename",
"edit-meta-data": "Edit meta data",
"creator": "Creator",
"contributors": "Contributors",
"subject": "Subject",
"description": "Description",
"row-number": "Row&nbsp;Number",
"last-mod": "Last&nbsp;modified",
"del-title": "Delete this project",
"del-body": "Are you sure you want to delete project \"",

View File

@ -202,6 +202,7 @@ $(function() {
Refine.selectActionArea('create-project');
$("#slogan").text($.i18n._('core-index')["slogan"]+".");
$("#or-index-pref").text($.i18n._('core-index')["preferences"]);
$("#or-index-help").text($.i18n._('core-index')["help"]);
$("#or-index-about").text($.i18n._('core-index')["about"]);
$("#or-index-noProj").text($.i18n._('core-index')["no-proj"]+".");

View File

@ -0,0 +1,98 @@
function EditMetadataDialog(metaData, targetRowElem) {
this._metaDataUIs = [];
this._metaData = metaData;
this._MetaDataUI = function(tr, key, value, project) {
var self = this;
var td0 = tr.insertCell(0);
$(td0).text(key);
var td1 = tr.insertCell(1);
$(td1).text((value !== null) ? value : "");
var td2 = tr.insertCell(2);
$('<button class="button">').text($.i18n._('core-index')["edit"]).appendTo(td2).click(function() {
var newValue = window.prompt($.i18n._('core-index')["change-metadata-value"]+" " + key, value);
if (newValue !== null) {
$(td1).text(newValue);
metaData[key] = newValue;
$.post(
"command/core/set-metaData",
{
project : project,
name : key,
value : newValue
},
function(o) {
if (o.code === "error") {
alert(o.message);
}
},
"json"
);
}
Refine.OpenProjectUI.refreshProject(targetRowElem, metaData);
});
};
this._createDialog();
}
EditMetadataDialog.prototype._createDialog = function() {
var self = this;
var frame = $(DOM.loadHTML("core", "scripts/project/edit-metadata-dialog.html"));
this._elmts = DOM.bind(frame);
this._level = DialogSystem.showDialog(frame);
this._elmts.closeButton.html($.i18n._('core-buttons')["close"]);
this._elmts.closeButton.click(function() { self._dismiss(); });
var body = $("#metadata-body");
$('<h1>').text($.i18n._('core-index')["metaDatas"]).appendTo(body);
var metadataTable = $("<table>")
.addClass("list-table")
.addClass("preferences")
.html('<tr><th>'+$.i18n._('core-index')["key"]+'</th><th>'+$.i18n._('core-index')["value"]+'</th><th></th></tr>')
.appendTo(body)[0];
var flattenObject = function(ob, key) {
var toReturn = {};
for ( var i in ob) {
if (i !== key) {
toReturn[i] = ob[i];
continue;
}
for ( var x in ob[i]) {
toReturn[ob[i][x].name] = ob[i][x].value;
}
}
return toReturn;
};
var flatMetaData = flattenObject(this._metaData, "userMetaData");
for (var k in flatMetaData) {
var tr = metadataTable.insertRow(metadataTable.rows.length);
var v;
if (typeof flatMetaData[k] === 'string') {
v = flatMetaData[k].replace(/\"/g, "");
} else {
v = JSON.stringify(flatMetaData[k]);
}
this._metaDataUIs.push(new this._MetaDataUI(tr, k, v, flatMetaData.id));
}
};
EditMetadataDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};

View File

@ -120,16 +120,31 @@ Refine.OpenProjectUI.prototype._renderProjects = function(data) {
Refine.selectActionArea('open-project');
var table = $(
'<table class="list-table"><tr>' +
'<table class="tablesorter-blue list-table"><thead><tr>' +
'<th></th>' +
'<th></th>' +
'<th></th>' +
'<th>'+$.i18n._('core-index-open')["last-mod"]+'</th>' +
'<th>'+$.i18n._('core-index-open')["name"]+'</th>' +
'</tr></table>'
'<th>'+$.i18n._('core-index-open')["creator"]+'</th>' +
'<th>'+$.i18n._('core-index-open')["subject"]+'</th>' +
'<th>'+$.i18n._('core-index-open')["description"]+'</th>' +
'<th>'+$.i18n._('core-index-open')["row-number"]+'</th>' +
(function() {
var htmlDisplay = "";
for (var n in data.customMetaDataColumns) {
if (data.customMetaDataColumns[n].display) {
htmlDisplay += '<th>'+ data.customMetaDataColumns[n].name + '</th>';
}
}
return htmlDisplay;
})() +
'</tr></thead><tbody></tbody></table>'
).appendTo(container)[0];
var renderProject = function(project) {
var tr = table.insertRow(table.rows.length);
var tr = table.getElementsByTagName('tbody')[0].insertRow(table.rows.length - 1);
tr.className = "project";
var deleteLink = $('<a></a>')
@ -189,31 +204,67 @@ Refine.OpenProjectUI.prototype._renderProjects = function(data) {
}).appendTo(
$(tr.insertCell(tr.cells.length)).css('width', '1%')
);
var editMetaDataLink = $('<a></a>')
.text($.i18n._('core-index-open')["edit-meta-data"])
.addClass("secondary")
.attr("href", "javascript:{}")
.css("visibility", "hidden")
.click(function() {
new EditMetadataDialog(project, $(this).parent().parent());
})
.appendTo(
$(tr.insertCell(tr.cells.length)).css('width', '3%')
);
$('<div></div>')
.html(formatRelativeDate(project.date))
.addClass("last-modified")
.attr("title", project.date.toString())
.appendTo($(tr.insertCell(tr.cells.length)).attr('width', '1%'));
var nameLink = $('<a></a>')
.addClass("project-name")
.text(project.name)
.attr("href", "project?project=" + project.id)
.appendTo(tr.insertCell(tr.cells.length));
.appendTo($(tr.insertCell(tr.cells.length)).attr('width', '10%'));
var appendMetaField = function(data, width) {
width = width || '1%';
$('<div></div>')
.html(data)
.appendTo($(tr.insertCell(tr.cells.length)).attr('width', width));
};
appendMetaField(project.creator);
appendMetaField(project.subject);
appendMetaField(project.description, '20%');
appendMetaField(project.rowNumber);
var data = project.userMetaData;
for(var i in data)
{
if (data[i].display === true) {
appendMetaField(data[i].value);
}
}
$(tr).mouseenter(function() {
renameLink.css("visibility", "visible");
deleteLink.css("visibility", "visible");
editMetaDataLink.css("visibility", "visible");
}).mouseleave(function() {
renameLink.css("visibility", "hidden");
deleteLink.css("visibility", "hidden");
editMetaDataLink.css("visibility", "hidden");
});
};
for (var i = 0; i < projects.length; i++) {
renderProject(projects[i]);
}
$(table).tablesorter();
}
};
@ -247,6 +298,43 @@ Refine.OpenProjectUI.prototype._onClickUploadFileButton = function(evt) {
return false;
};
Refine.OpenProjectUI.refreshProject = function(tr, metaData) {
var refreshMetaField = function(data, index) {
$('td', tr).eq(index)
.html(data);
};
var index = 5;
refreshMetaField(metaData.creator, index); index++;
refreshMetaField(metaData.subject,index); index++;
refreshMetaField(metaData.description,index); index++;
refreshMetaField(metaData.rowNumbe,index); index++;
var updateUserMetaData = function(ob) {
var userMetaData = ob.userMetaData;
for ( var n in ob) {
for ( var i in userMetaData) {
if (n === userMetaData[i].name) {
userMetaData[i].value = ob[n];
break;
}
}
}
ob.userMetaData = userMetaData;
};
updateUserMetaData(metaData);
var data = metaData.userMetaData;
for(var i in data)
{
if (data[i].display === true) {
refreshMetaField(data[i].value,index); index++;
}
}
};
Refine.actionAreas.push({
id: "open-project",
label: $.i18n._('core-index-open')["open-proj"],

View File

@ -0,0 +1,11 @@
<div class="dialog-frame" style="width: 1600px; height: 700px">
<div class="dialog-border">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div id="metadata-body" class="grid-layout layout-normal layout-full"></div>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button class="button" bind="closeButton"></button>
</div>
</div>
</div>

View File

@ -1,4 +1,4 @@
<div class="dialog-frame" style="width: 700px;">
<div class="dialog-frame" >
<div class="dialog-border">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">

View File

@ -87,3 +87,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
opacity: 0.2;
}
#metadata-body tr:nth-child(even) {background: #CCC}
#metadata-body tr:nth-child(odd) {background: #FFF}