c4b0ff6bea
* fix the appbundle issue #1209 * fix #1162 allow the JRE 9 * fix the package declarations * remove the _ from the method name * use the explicit scoping * remote extra ; * fix issued from codacy * fix issued from codacy * add preferences link to the index page * handle the empty user metadata * fix 'last modified' sorting issue #1307 * prevent overflow of the table. issue #1306 * add isoDateParser to sort the date * prevent overflow of the project index * remove sorter arrow for action columns * disable editing the internal metadata * adjust the width of the table * change MetaData to Metadata * change the filed name from rowNumber to rowCount * put back the incidently deleted gitignore * add double quote to prevent word splitting * UI improvement on metadata view and project list view * remove the date field in metadata * message notification of the free RAM. Issue #1295 * UI tuning for metadata view * shorten the ISO date to locale date format * Added translation using Weblate (Portuguese (Brazil)) * remove the rename link * Ignore empty language files introduced by Weblate * Add UI for Invert text filter * Backend support for Inverting Text search facets * Fix reset on text search facet * More succinct return statements * add tests for SetProjectMetadataCommand * Tidying up for Codacy * Added Tests for TextSearchFilter * Corrections for Codacy * More code tidy up * let the browser auto fit the table cell when resizing/zooming * fix import multiple excel with mulitple sheets issue #1328 * check if the project has the userMetadata * fix the unit test support multi files with multi tables for open office * prevent the same key for user metadata * replace _ with variable for exception * fix the no-undef issue * to adjust the width of transform dialog. issue #1332 * fix the row count refresh issue * extract method * move the log message * cosmatic changes for codacy * fix typo * bump to version 2.8 * .gitignore is now working * preview stage won't have the metadata populated, so protect NPE * Update README.md No more direct link to the last version tag, which will avoid having to think of updating the readme * refacotring the ProjectMetadata class * introduce the IMetadata interface * create submodule of dataschema * add back * setup lib for dataschema; upgrade the apache lang to lang3 * replace escape* functions from apache lang3 * replace the ProjectMetadata with IMetadata interface * add missing jars * set the IMetadata a field of Project * remove PreferenceStore out of Project model * fix test SetProjectMetadataCommandTests by casting * introdcue the AbstractMetadata * introdcue the AbstractMetadata * reorganize the metadata package * allow have mulitiple metadata for a project * support for mulitple metadata format * remove jdk7 since 'table schema' java implmentation only support jdk8+ * set execute permission for script * fix the Unit Test after Metadata refactoring * restore the apache lang2.5 since jetty 6.1.22 depend on it * add commons lang 2.5 jar * git submodule add https://github.com/frictionlessdata/datapackage-java * remove the metadata parameter from the ProjectManager.registerProject method * remove hashmap _projectsMetadata field from the ProjectManager and FileProjectManager * init the Project.metadataMap * fix Unit Test * restore the ProjectMetaData map to ProjectManager * put the ProjectMetaDta in place for ProjectManager and Project object * check null of singleton instead of create a constructor just for test * load the data package metadata * importing data package * importing data package * encapsulate the Package class into DataPackageMetadata * user _ to indicate the class fields * introduce base URL in order to download the data files * import data package UI and draft backend * import data package UI * fix typo * download the data set pointed from metadata resource * save and load the data package metadata * avoid magic string * package cleanup * set the java_version to 1.8 * set the min jdk to 1.8 * add the 3rd party src in the build.xml * skip the file selection page if only 1 DATA file * add files structure for json editor * seperate out the metadata file from the retrival file list * rename the OKF_METADATA to DATAPACKAGE_METADATA * clean up * implement GetMetadateCommand class * display the metadata in json format * git submodule update --remote --merge * adjust the setting after pulling from datapackage origin * fix the failed UT DateExtensionTests.testFetchCounts due to new json jar json-20160810.jar will complain: JSONObject["float"] not a string. * clean up the weird loop array syntax get complained * remove the unused constant * export in data package format * interface cleanup * fix UT * edit the metadata * add UT for SetMetadataCommand * fix UT for SetMetadataCommand * display the data package metadata link on the project index page * update submodule * log the exceptions * Ajv does not work properly, use the back end validation instead * enable the validation for jsoneditor * first draft of the data validation * create a map to hold the constraint and its handler * rename * support for minLength and maxLength from spec * add validate command * test the opeation instead of validate command * rename the UT * format the error message and push to the report * fix row number * add resource bundle for validator * inject the code of the constrains * make the StrSubstitutor works * extract the type and format information * add the customizedFormat to interface to allow format properly * get rid of magic string * take care of missing parts of the data package * implement RequiredConstraint * patch for number type * add max/min constraints * get the constrains directly from field * implement the PatternConstraint * suppress warning * fix the broken UT when expecting 2 digits fraction * handle the cast and type properly * fix the missing resource files for data package when run from command line * use the copy instead of copydir * add script for appveyor * update script for appveyor * do recursive clone * correct the git url * fix clone path * clone folder option does not work * will put another PR for this. delete for now * revert the interface method name * lazy loading the project data * disable the validate menu for now * add UT * assert UTs * add UT * fix #1386 * remove import * test the thread * Revert "test the thread" This reverts commit 779214160055afe3ccdcc18c57b0c7c72e87c824. * fix the URLCachingTest UT * define the template data package * tidy up the metadata interface * check the http response code * fix the package * display user friendly message when URL path is not reachable * populate the data package schema * Delete hs_err_pid15194.log * populate data package info * add username preference and it will be pulled as the creator of the metadata * undo the project.updateColumnChange() and start to introduce the fields into the existing core model * tightly integrate the data package metadata * tightly integrate the data package metadata for project level * remove the submodule * move the edit botton * clean up build * load the new property * load the project metadata * fix issues from codacy * remove unused fields and annotation * check the http response code firstly * import zipped data package * allow without keywords * process the zip data package from url * merge the tags * check store firstly * remove the table schema src * move the json schema files to schema dir * add comment * add comment * remove git moduels * add incidently deleted file * fix typo * remove SetMetadataCommand * revert change * merge from master
393 lines
13 KiB
Java
393 lines
13 KiB
Java
/*
|
|
|
|
Copyright 2010, Google Inc.
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are
|
|
met:
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
* Redistributions in binary form must reproduce the above
|
|
copyright notice, this list of conditions and the following disclaimer
|
|
in the documentation and/or other materials provided with the
|
|
distribution.
|
|
* Neither the name of Google Inc. nor the names of its
|
|
contributors may be used to endorse or promote products derived from
|
|
this software without specific prior written permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
package com.google.refine.commands;
|
|
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.io.StringWriter;
|
|
import java.io.Writer;
|
|
import java.util.Properties;
|
|
|
|
import javax.servlet.ServletException;
|
|
import javax.servlet.http.HttpServletRequest;
|
|
import javax.servlet.http.HttpServletResponse;
|
|
|
|
import org.apache.velocity.VelocityContext;
|
|
import org.json.JSONException;
|
|
import org.json.JSONObject;
|
|
import org.json.JSONWriter;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import com.google.refine.Jsonizable;
|
|
import com.google.refine.ProjectManager;
|
|
import com.google.refine.RefineServlet;
|
|
import com.google.refine.browsing.Engine;
|
|
import com.google.refine.history.HistoryEntry;
|
|
import com.google.refine.model.Project;
|
|
import com.google.refine.model.medadata.ProjectMetadata;
|
|
import com.google.refine.process.Process;
|
|
import com.google.refine.util.ParsingUtilities;
|
|
|
|
/**
|
|
* The super class of all calls that the client side can invoke, most of which
|
|
* are AJAX calls.
|
|
*/
|
|
public abstract class Command {
|
|
|
|
final static protected Logger logger = LoggerFactory.getLogger("command");
|
|
|
|
protected RefineServlet servlet;
|
|
|
|
public void init(RefineServlet servlet) {
|
|
this.servlet = servlet;
|
|
}
|
|
|
|
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
|
throws ServletException, IOException {
|
|
|
|
throw new UnsupportedOperationException();
|
|
};
|
|
|
|
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
|
throws ServletException, IOException {
|
|
|
|
throw new UnsupportedOperationException();
|
|
};
|
|
|
|
public void doPut(HttpServletRequest request, HttpServletResponse response)
|
|
throws ServletException, IOException {
|
|
|
|
throw new UnsupportedOperationException();
|
|
};
|
|
|
|
public void doDelete(HttpServletRequest request, HttpServletResponse response)
|
|
throws ServletException, IOException {
|
|
|
|
throw new UnsupportedOperationException();
|
|
};
|
|
|
|
/**
|
|
* Whether each request to this command should be logged. For some commands
|
|
* that can get called too frequently, such as GetProcessesCommand, logging
|
|
* is very distracting.
|
|
*/
|
|
public boolean logRequests() {
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Utility function to get the browsing engine's configuration as a JSON object
|
|
* from the "engine" request parameter, most often in the POST body.
|
|
*
|
|
* @param request
|
|
* @return
|
|
* @throws JSONException
|
|
*/
|
|
static protected JSONObject getEngineConfig(HttpServletRequest request)
|
|
throws JSONException {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
|
|
String json = request.getParameter("engine");
|
|
try{
|
|
return (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
|
|
} catch (JSONException e){
|
|
logger.debug( json + " could not be parsed to JSON");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Utility function to reconstruct the browsing engine from the "engine" request parameter,
|
|
* most often in the POST body.
|
|
*
|
|
* @param request
|
|
* @param project
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
static protected Engine getEngine(HttpServletRequest request, Project project)
|
|
throws Exception {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
if (project == null) {
|
|
throw new IllegalArgumentException("parameter 'project' should not be null");
|
|
}
|
|
|
|
Engine engine = new Engine(project);
|
|
JSONObject o = getEngineConfig(request);
|
|
if (o != null) {
|
|
engine.initializeFromJSON(o);
|
|
}
|
|
return engine;
|
|
}
|
|
|
|
/**
|
|
* Utility method for retrieving the Project object having the ID specified
|
|
* in the "project" URL parameter.
|
|
*
|
|
* @param request
|
|
* @return
|
|
* @throws ServletException
|
|
*/
|
|
protected Project getProject(HttpServletRequest request) throws ServletException {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
String param = request.getParameter("project");
|
|
if (param == null || "".equals(param)) {
|
|
throw new ServletException("Can't find project: missing ID parameter");
|
|
}
|
|
Long id;
|
|
try {
|
|
id = Long.parseLong(param);
|
|
} catch (NumberFormatException e) {
|
|
throw new ServletException("Can't find project: badly formatted id #", e);
|
|
}
|
|
Project p = ProjectManager.singleton.getProject(id);
|
|
if (p != null) {
|
|
return p;
|
|
} else {
|
|
throw new ServletException("Failed to find project id #" + param + " - may be corrupt");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Utility method for retrieving the ProjectMetadata object having the ID specified
|
|
* in the "project" URL parameter.
|
|
*
|
|
* @param request
|
|
* @return
|
|
* @throws ServletException
|
|
*/
|
|
protected ProjectMetadata getMetadata(HttpServletRequest request) throws ServletException {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
try {
|
|
ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(Long.parseLong(request.getParameter("project")));
|
|
if (pm != null) {
|
|
return pm;
|
|
}
|
|
} catch (Exception e) {
|
|
// ignore
|
|
}
|
|
throw new ServletException("Can't find project metadata: missing or bad URL parameter");
|
|
}
|
|
|
|
static protected int getIntegerParameter(HttpServletRequest request, String name, int def) {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
try {
|
|
return Integer.parseInt(request.getParameter(name));
|
|
} catch (Exception e) {
|
|
// ignore
|
|
}
|
|
return def;
|
|
}
|
|
|
|
static protected JSONObject getJsonParameter(HttpServletRequest request, String name) {
|
|
if (request == null) {
|
|
throw new IllegalArgumentException("parameter 'request' should not be null");
|
|
}
|
|
String value = request.getParameter(name);
|
|
if (value != null) {
|
|
try {
|
|
return ParsingUtilities.evaluateJsonStringToObject(value);
|
|
} catch (JSONException e) {
|
|
logger.warn("error getting json parameter",e);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
static protected void performProcessAndRespond(
|
|
HttpServletRequest request,
|
|
HttpServletResponse response,
|
|
Project project,
|
|
Process process
|
|
) throws Exception {
|
|
response.setCharacterEncoding("UTF-8");
|
|
response.setHeader("Content-Type", "application/json");
|
|
|
|
HistoryEntry historyEntry = project.processManager.queueProcess(process);
|
|
if (historyEntry != null) {
|
|
Writer w = response.getWriter();
|
|
JSONWriter writer = new JSONWriter(w);
|
|
Properties options = new Properties();
|
|
|
|
writer.object();
|
|
writer.key("code"); writer.value("ok");
|
|
writer.key("historyEntry"); historyEntry.write(writer, options);
|
|
writer.endObject();
|
|
|
|
w.flush();
|
|
w.close();
|
|
} else {
|
|
respond(response, "{ \"code\" : \"pending\" }");
|
|
}
|
|
}
|
|
|
|
static protected void respond(HttpServletResponse response, String content)
|
|
throws IOException, ServletException {
|
|
|
|
response.setCharacterEncoding("UTF-8");
|
|
response.setStatus(HttpServletResponse.SC_OK);
|
|
Writer w = response.getWriter();
|
|
if (w != null) {
|
|
w.write(content);
|
|
w.flush();
|
|
w.close();
|
|
} else {
|
|
throw new ServletException("response returned a null writer");
|
|
}
|
|
}
|
|
|
|
static protected void respond(HttpServletResponse response, String status, String message)
|
|
throws IOException, JSONException {
|
|
|
|
Writer w = response.getWriter();
|
|
JSONWriter writer = new JSONWriter(w);
|
|
writer.object();
|
|
writer.key("status"); writer.value(status);
|
|
writer.key("message"); writer.value(message);
|
|
writer.endObject();
|
|
w.flush();
|
|
w.close();
|
|
}
|
|
|
|
static protected void respondJSON(HttpServletResponse response, Jsonizable o)
|
|
throws IOException, JSONException {
|
|
|
|
respondJSON(response, o, new Properties());
|
|
}
|
|
|
|
static protected void respondJSON(
|
|
HttpServletResponse response, Jsonizable o, Properties options)
|
|
throws IOException, JSONException {
|
|
|
|
response.setCharacterEncoding("UTF-8");
|
|
response.setHeader("Content-Type", "application/json");
|
|
response.setHeader("Cache-Control", "no-cache");
|
|
|
|
Writer w = response.getWriter();
|
|
JSONWriter writer = new JSONWriter(w);
|
|
|
|
o.write(writer, options);
|
|
w.flush();
|
|
w.close();
|
|
}
|
|
|
|
static protected void respondJSONObject(
|
|
HttpServletResponse response, JSONObject o)
|
|
throws IOException, JSONException {
|
|
response.setCharacterEncoding("UTF-8");
|
|
response.setHeader("Content-Type", "application/json");
|
|
response.setHeader("Cache-Control", "no-cache");
|
|
|
|
Writer w = response.getWriter();
|
|
w.append(o.toString());
|
|
w.flush();
|
|
w.close();
|
|
}
|
|
|
|
static protected void respondException(HttpServletResponse response, Exception e)
|
|
throws IOException, ServletException {
|
|
|
|
logger.warn("Exception caught", e);
|
|
|
|
if (response == null) {
|
|
throw new ServletException("Response object can't be null");
|
|
}
|
|
|
|
try {
|
|
JSONObject o = new JSONObject();
|
|
o.put("code", "error");
|
|
o.put("message", e.getMessage());
|
|
|
|
StringWriter sw = new StringWriter();
|
|
PrintWriter pw = new PrintWriter(sw);
|
|
e.printStackTrace(pw);
|
|
pw.flush();
|
|
sw.flush();
|
|
|
|
o.put("stack", sw.toString());
|
|
|
|
response.setCharacterEncoding("UTF-8");
|
|
response.setHeader("Content-Type", "application/json");
|
|
respond(response, o.toString());
|
|
} catch (JSONException e1) {
|
|
e.printStackTrace(response.getWriter());
|
|
}
|
|
}
|
|
|
|
protected void respondWithErrorPage(
|
|
HttpServletRequest request,
|
|
HttpServletResponse response,
|
|
String message,
|
|
Throwable e
|
|
) {
|
|
VelocityContext context = new VelocityContext();
|
|
|
|
context.put("message", message);
|
|
|
|
if (e != null) {
|
|
StringWriter writer = new StringWriter();
|
|
|
|
e.printStackTrace(new PrintWriter(writer));
|
|
|
|
context.put("stack", writer.toString());
|
|
} else {
|
|
context.put("stack", "");
|
|
}
|
|
|
|
try {
|
|
servlet.getModule("core").sendTextFromTemplate(
|
|
request, response, context, "error.vt", "UTF-8", "text/html", true);
|
|
|
|
} catch (Exception e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
}
|
|
|
|
static protected void redirect(HttpServletResponse response, String url) throws IOException {
|
|
response.sendRedirect(url);
|
|
}
|
|
|
|
}
|