added null and default value options
This commit is contained in:
parent
c7c0d8884a
commit
0eb7b4082f
@ -1,88 +0,0 @@
|
|||||||
package com.google.refine.commands.exporting;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Enumeration;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.google.refine.ProjectManager;
|
|
||||||
import com.google.refine.commands.Command;
|
|
||||||
import com.google.refine.exporters.sql.SqlDataError;
|
|
||||||
import com.google.refine.exporters.sql.SqlExporter;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
|
|
||||||
|
|
||||||
public class SqlExporterCommand extends Command {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger("SqlExporterCommand");
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
public Properties getRequestParameters(HttpServletRequest request) {
|
|
||||||
Properties options = new Properties();
|
|
||||||
Enumeration<String> en = request.getParameterNames();
|
|
||||||
while (en.hasMoreElements()) {
|
|
||||||
String name = en.nextElement();
|
|
||||||
String value = request.getParameter(name);
|
|
||||||
options.put(name, value);
|
|
||||||
|
|
||||||
}
|
|
||||||
return options;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
|
||||||
throws ServletException, IOException {
|
|
||||||
|
|
||||||
ProjectManager.singleton.setBusy(true);
|
|
||||||
try {
|
|
||||||
response.setCharacterEncoding("UTF-8");
|
|
||||||
response.setHeader("Content-Type", "application/json");
|
|
||||||
JSONWriter writer = new JSONWriter(response.getWriter());
|
|
||||||
writer.object();
|
|
||||||
Project project = getProject(request);
|
|
||||||
//Engine engine = getEngine(request, project);
|
|
||||||
Properties params = getRequestParameters(request);
|
|
||||||
SqlExporter sqlExporter = new SqlExporter();
|
|
||||||
List<SqlDataError> result = sqlExporter.validateSqlData(project, params);
|
|
||||||
if(result == null || result.isEmpty()) {
|
|
||||||
writer.key("code");
|
|
||||||
writer.value("OK");
|
|
||||||
}else {
|
|
||||||
writer.key("code");
|
|
||||||
writer.value("Error");
|
|
||||||
|
|
||||||
writer.key("SqlDataErrors");
|
|
||||||
writer.array();
|
|
||||||
result.stream().forEach(err -> {
|
|
||||||
writer.object();
|
|
||||||
|
|
||||||
writer.key("errorcode");
|
|
||||||
writer.value(err.getErrorCode());
|
|
||||||
writer.key("errormessage");
|
|
||||||
writer.value(err.getErrorMessage());
|
|
||||||
|
|
||||||
writer.endObject();
|
|
||||||
|
|
||||||
});
|
|
||||||
writer.endArray();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.endObject();
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new ServletException(e);
|
|
||||||
} finally {
|
|
||||||
ProjectManager.singleton.setBusy(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
@ -44,6 +44,10 @@ import javax.servlet.ServletException;
|
|||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
|
import org.apache.http.HttpStatus;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.google.refine.ProjectManager;
|
import com.google.refine.ProjectManager;
|
||||||
import com.google.refine.browsing.Engine;
|
import com.google.refine.browsing.Engine;
|
||||||
import com.google.refine.commands.Command;
|
import com.google.refine.commands.Command;
|
||||||
@ -52,9 +56,12 @@ import com.google.refine.exporters.Exporter;
|
|||||||
import com.google.refine.exporters.ExporterRegistry;
|
import com.google.refine.exporters.ExporterRegistry;
|
||||||
import com.google.refine.exporters.StreamExporter;
|
import com.google.refine.exporters.StreamExporter;
|
||||||
import com.google.refine.exporters.WriterExporter;
|
import com.google.refine.exporters.WriterExporter;
|
||||||
|
import com.google.refine.exporters.sql.SqlExporterException;
|
||||||
|
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
|
|
||||||
public class ExportRowsCommand extends Command {
|
public class ExportRowsCommand extends Command {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger("ExportRowsCommand");
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
static public Properties getRequestParameters(HttpServletRequest request) {
|
static public Properties getRequestParameters(HttpServletRequest request) {
|
||||||
@ -73,6 +80,7 @@ public class ExportRowsCommand extends Command {
|
|||||||
throws ServletException, IOException {
|
throws ServletException, IOException {
|
||||||
|
|
||||||
ProjectManager.singleton.setBusy(true);
|
ProjectManager.singleton.setBusy(true);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Project project = getProject(request);
|
Project project = getProject(request);
|
||||||
Engine engine = getEngine(request, project);
|
Engine engine = getEngine(request, project);
|
||||||
@ -100,7 +108,8 @@ public class ExportRowsCommand extends Command {
|
|||||||
|
|
||||||
((WriterExporter) exporter).export(project, params, engine, writer);
|
((WriterExporter) exporter).export(project, params, engine, writer);
|
||||||
writer.close();
|
writer.close();
|
||||||
} else if (exporter instanceof StreamExporter) {
|
}
|
||||||
|
else if (exporter instanceof StreamExporter) {
|
||||||
response.setCharacterEncoding("UTF-8");
|
response.setCharacterEncoding("UTF-8");
|
||||||
|
|
||||||
OutputStream stream = response.getOutputStream();
|
OutputStream stream = response.getOutputStream();
|
||||||
@ -108,12 +117,17 @@ public class ExportRowsCommand extends Command {
|
|||||||
stream.close();
|
stream.close();
|
||||||
// } else if (exporter instanceof UrlExporter) {
|
// } else if (exporter instanceof UrlExporter) {
|
||||||
// ((UrlExporter) exporter).export(project, options, engine);
|
// ((UrlExporter) exporter).export(project, options, engine);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// TODO: Should this use ServletException instead of respondException?
|
// TODO: Should this use ServletException instead of respondException?
|
||||||
respondException(response, new RuntimeException("Unknown exporter type"));
|
respondException(response, new RuntimeException("Unknown exporter type"));
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// Use generic error handling rather than our JSON handling
|
// Use generic error handling rather than our JSON handling
|
||||||
|
logger.info("error:{}", e.getMessage());
|
||||||
|
if (e instanceof SqlExporterException) {
|
||||||
|
response.sendError(HttpStatus.SC_BAD_REQUEST, e.getMessage());
|
||||||
|
}
|
||||||
throw new ServletException(e);
|
throw new ServletException(e);
|
||||||
} finally {
|
} finally {
|
||||||
ProjectManager.singleton.setBusy(false);
|
ProjectManager.singleton.setBusy(false);
|
||||||
|
@ -229,12 +229,16 @@ abstract public class CustomizableTabularExporterUtilities {
|
|||||||
|
|
||||||
Map<String, String> identifierSpaceToUrl = null;
|
Map<String, String> identifierSpaceToUrl = null;
|
||||||
|
|
||||||
|
//SQLExporter parameter to convert null cell value to empty string
|
||||||
|
boolean includeNullFieldValue = false;
|
||||||
|
|
||||||
CellFormatter() {
|
CellFormatter() {
|
||||||
dateFormatter = new SimpleDateFormat(fullIso8601);
|
dateFormatter = new SimpleDateFormat(fullIso8601);
|
||||||
}
|
}
|
||||||
|
|
||||||
CellFormatter(JSONObject options) {
|
CellFormatter(JSONObject options) {
|
||||||
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings");
|
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings");
|
||||||
|
includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false);
|
||||||
if (reconSettings != null) {
|
if (reconSettings != null) {
|
||||||
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null);
|
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null);
|
||||||
if ("entity-name".equals(reconOutputString)) {
|
if ("entity-name".equals(reconOutputString)) {
|
||||||
@ -354,6 +358,12 @@ abstract public class CustomizableTabularExporterUtilities {
|
|||||||
}
|
}
|
||||||
return new CellData(column.getName(), value, text, link);
|
return new CellData(column.getName(), value, text, link);
|
||||||
}
|
}
|
||||||
|
}else {//added for sql exporter
|
||||||
|
|
||||||
|
if(includeNullFieldValue) {
|
||||||
|
return new CellData(column.getName(), "", "", "");
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -384,4 +394,4 @@ abstract public class CustomizableTabularExporterUtilities {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -43,16 +43,15 @@ public class SqlCreateBuilder {
|
|||||||
private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder");
|
private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder");
|
||||||
|
|
||||||
private String table;
|
private String table;
|
||||||
|
|
||||||
|
|
||||||
private List<String> columns;
|
private List<String> columns;
|
||||||
|
|
||||||
private JSONObject options;
|
private JSONObject options;
|
||||||
|
|
||||||
|
|
||||||
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) {
|
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) {
|
||||||
this.table = table;
|
this.table = table;
|
||||||
this.columns = columns;
|
this.columns = columns;
|
||||||
this.options = options;
|
this.options = options;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getCreateSQL() {
|
public String getCreateSQL() {
|
||||||
@ -76,6 +75,7 @@ public class SqlCreateBuilder {
|
|||||||
String size = JSONUtilities.getString(columnOptions, "size", "");
|
String size = JSONUtilities.getString(columnOptions, "size", "");
|
||||||
boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true);
|
boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true);
|
||||||
String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null);
|
String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null);
|
||||||
|
logger.info("allowNull::{}" , allowNull);
|
||||||
|
|
||||||
String allowNullStr = "NULL";
|
String allowNullStr = "NULL";
|
||||||
if(!allowNull) {
|
if(!allowNull) {
|
||||||
@ -125,6 +125,11 @@ public class SqlCreateBuilder {
|
|||||||
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
|
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
|
||||||
createSB.append(" DEFAULT " + "'" + defaultValue + "'");
|
createSB.append(" DEFAULT " + "'" + defaultValue + "'");
|
||||||
}else {
|
}else {
|
||||||
|
try {
|
||||||
|
Integer.parseInt(defaultValue);
|
||||||
|
}catch(NumberFormatException nfe) {
|
||||||
|
throw new SqlExporterException(defaultValue + " is not compatible with column type :" + type);
|
||||||
|
}
|
||||||
createSB.append(" DEFAULT " + defaultValue);
|
createSB.append(" DEFAULT " + defaultValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,6 +41,8 @@ public class SqlData {
|
|||||||
public static final String SQL_TYPE_INTEGER = "INTEGER";
|
public static final String SQL_TYPE_INTEGER = "INTEGER";
|
||||||
public static final String SQL_TYPE_INT = "INT";
|
public static final String SQL_TYPE_INT = "INT";
|
||||||
public static final String SQL_TYPE_NUMERIC = "NUMERIC";
|
public static final String SQL_TYPE_NUMERIC = "NUMERIC";
|
||||||
|
public static final String SQL_TYPE_DATE = "DATE";
|
||||||
|
public static final String SQL_TYPE_TIMESTAMP = "TIMESTAMP";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,34 +0,0 @@
|
|||||||
package com.google.refine.exporters.sql;
|
|
||||||
|
|
||||||
|
|
||||||
public class SqlDataError {
|
|
||||||
private int errorCode;
|
|
||||||
private String errorMessage;
|
|
||||||
|
|
||||||
public SqlDataError(int errorCode, String errorMessage) {
|
|
||||||
super();
|
|
||||||
this.errorCode = errorCode;
|
|
||||||
this.errorMessage = errorMessage;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int getErrorCode() {
|
|
||||||
return errorCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void setErrorCode(int errorCode) {
|
|
||||||
this.errorCode = errorCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public String getErrorMessage() {
|
|
||||||
return errorMessage;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void setErrorMessage(String errorMessage) {
|
|
||||||
this.errorMessage = errorMessage;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
@ -60,6 +60,7 @@ public class SqlExporter implements WriterExporter {
|
|||||||
private List<String> columnNames = new ArrayList<String>();
|
private List<String> columnNames = new ArrayList<String>();
|
||||||
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
|
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
|
||||||
private JSONObject sqlOptions;
|
private JSONObject sqlOptions;
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getContentType() {
|
public String getContentType() {
|
||||||
@ -72,6 +73,7 @@ public class SqlExporter implements WriterExporter {
|
|||||||
if(logger.isDebugEnabled()) {
|
if(logger.isDebugEnabled()) {
|
||||||
logger.debug("export sql with params: {}", params);
|
logger.debug("export sql with params: {}", params);
|
||||||
}
|
}
|
||||||
|
|
||||||
TabularSerializer serializer = new TabularSerializer() {
|
TabularSerializer serializer = new TabularSerializer() {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -84,14 +86,14 @@ public class SqlExporter implements WriterExporter {
|
|||||||
public void endFile() {
|
public void endFile() {
|
||||||
try {
|
try {
|
||||||
if (columnNames.isEmpty()) {
|
if (columnNames.isEmpty()) {
|
||||||
writer.write(NO_COL_SELECTED_ERROR);
|
|
||||||
logger.error("No Columns Selected!!");
|
logger.error("No Columns Selected!!");
|
||||||
return;
|
throw new SqlExporterException(NO_COL_SELECTED_ERROR);
|
||||||
|
|
||||||
}
|
}
|
||||||
if (sqlOptions == null) {
|
if (sqlOptions == null) {
|
||||||
writer.write(NO_OPTIONS_PRESENT_ERROR);
|
|
||||||
logger.error("No Options Selected!!");
|
logger.error("No Options Selected!!");
|
||||||
return;
|
throw new SqlExporterException(NO_OPTIONS_PRESENT_ERROR);
|
||||||
|
|
||||||
}
|
}
|
||||||
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
|
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
|
||||||
|
|
||||||
@ -163,11 +165,6 @@ public class SqlExporter implements WriterExporter {
|
|||||||
|
|
||||||
CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer);
|
CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer);
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<SqlDataError> validateSqlData(Project project, Properties params){
|
|
||||||
|
|
||||||
logger.info("Param Name:{}, Param Value:{}", project, params);
|
|
||||||
return null;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,16 @@
|
|||||||
|
package com.google.refine.exporters.sql;
|
||||||
|
|
||||||
|
|
||||||
|
public class SqlExporterException extends RuntimeException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -2640105469265805010L;
|
||||||
|
|
||||||
|
public SqlExporterException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -53,6 +53,7 @@ public class SqlInsertBuilder {
|
|||||||
private List<ArrayList<SqlData>> sqlDataList;
|
private List<ArrayList<SqlData>> sqlDataList;
|
||||||
|
|
||||||
private JSONObject options;
|
private JSONObject options;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
@ -60,12 +61,16 @@ public class SqlInsertBuilder {
|
|||||||
* @param columns
|
* @param columns
|
||||||
* @param rows
|
* @param rows
|
||||||
* @param options
|
* @param options
|
||||||
|
* @param sqlErrors
|
||||||
*/
|
*/
|
||||||
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options) {
|
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options
|
||||||
|
) {
|
||||||
this.table = table;
|
this.table = table;
|
||||||
this.columns = columns;
|
this.columns = columns;
|
||||||
this.sqlDataList = rows;
|
this.sqlDataList = rows;
|
||||||
this.options = options;
|
this.options = options;
|
||||||
|
//logger.info("Column Size:{}", columns.size());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -86,65 +91,74 @@ public class SqlInsertBuilder {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean nullValueToEmptyStr = options == null ? false : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true);
|
boolean nullValueNull = options == null ? true : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true);
|
||||||
|
|
||||||
StringBuffer values = new StringBuffer();
|
StringBuffer values = new StringBuffer();
|
||||||
|
|
||||||
int idx = 0;
|
int idx = 0;
|
||||||
for(ArrayList<SqlData> sqlRow : sqlDataList) {
|
for(ArrayList<SqlData> sqlRow : sqlDataList) {
|
||||||
StringBuilder rowValue = new StringBuilder();
|
StringBuilder rowValue = new StringBuilder();
|
||||||
|
|
||||||
|
//int fieldCount = 0;
|
||||||
for(SqlData val : sqlRow) {
|
for(SqlData val : sqlRow) {
|
||||||
|
|
||||||
JSONObject jsonOb = colOptionsMap.get(val.getColumnName());
|
JSONObject jsonOb = colOptionsMap.get(val.getColumnName());
|
||||||
String type = (String)jsonOb.get("type");
|
String type = (String)jsonOb.get("type");
|
||||||
String defaultValue = (String)jsonOb.get("defaultValue");
|
|
||||||
boolean allowNullChkBox = (boolean)jsonOb.get("allowNull");
|
|
||||||
|
|
||||||
|
|
||||||
|
String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null);
|
||||||
|
|
||||||
|
boolean allowNullChkBox = JSONUtilities.getBoolean(jsonOb, "defaultValue", true);;
|
||||||
if(type == null) {
|
if(type == null) {
|
||||||
type = SqlData.SQL_TYPE_VARCHAR;
|
type = SqlData.SQL_TYPE_VARCHAR;
|
||||||
}
|
}
|
||||||
//Character Types
|
//Character Types
|
||||||
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
|
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
|
||||||
|
|
||||||
if(!allowNullChkBox) {
|
if((val.getText() == null || val.getText().isEmpty()) ) {
|
||||||
throw new RuntimeException("bad input");
|
|
||||||
}
|
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
|
||||||
if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) {
|
|
||||||
// logger.info("Appending empty String:::{}" , val.getText());
|
|
||||||
if(defaultValue != null && !defaultValue.isEmpty()) {
|
|
||||||
rowValue.append("'" + defaultValue + "'");
|
|
||||||
}else {
|
|
||||||
rowValue.append("null");
|
|
||||||
}
|
|
||||||
|
|
||||||
}else {
|
}else {
|
||||||
rowValue.append("'" + val.getText() + "'");
|
rowValue.append("'" + val.getText() + "'");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}else {//Numeric Types
|
}else if(type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER) || type.equals(SqlData.SQL_TYPE_NUMERIC)) {//Numeric Types : INT, NUMERIC
|
||||||
|
|
||||||
if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) {
|
if((val.getText() == null || val.getText().isEmpty())) {
|
||||||
// logger.info("Appending empty String others:::{}" , val.getText());
|
|
||||||
if(defaultValue != null && !defaultValue.isEmpty()) {
|
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, false);
|
||||||
rowValue.append("'" + defaultValue + "'");
|
|
||||||
}else {
|
}else {//value not null
|
||||||
rowValue.append("null");
|
|
||||||
|
try {
|
||||||
|
Integer.parseInt(val.getText());
|
||||||
|
} catch (NumberFormatException nfe) {
|
||||||
|
throw new SqlExporterException(
|
||||||
|
val.getText() + " is not compatible with column type :" + type);
|
||||||
}
|
}
|
||||||
}else {
|
|
||||||
rowValue.append(val.getText());
|
rowValue.append(val.getText());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
}else if(type.equals(SqlData.SQL_TYPE_DATE) || type.equals(SqlData.SQL_TYPE_TIMESTAMP)) {
|
||||||
|
if((val.getText() == null || val.getText().isEmpty())) {
|
||||||
|
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
|
||||||
|
}else {
|
||||||
|
rowValue.append("'" + val.getText() + "'");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rowValue.append(",");
|
rowValue.append(",");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
idx++;
|
idx++;
|
||||||
String rowValString = rowValue.toString();
|
String rowValString = rowValue.toString();
|
||||||
|
// logger.info("rowValString::" + rowValString);
|
||||||
rowValString = rowValString.substring(0, rowValString.length() - 1);
|
rowValString = rowValString.substring(0, rowValString.length() - 1);
|
||||||
|
|
||||||
values.append("( ");
|
values.append("( ");
|
||||||
@ -182,4 +196,58 @@ public class SqlInsertBuilder {
|
|||||||
return sqlString;
|
return sqlString;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param allowNullChkBox
|
||||||
|
* @param defaultValue
|
||||||
|
* @param nullValueNull
|
||||||
|
* @param col
|
||||||
|
* @param rowValue
|
||||||
|
* @param quote
|
||||||
|
* @param fieldCount
|
||||||
|
*/
|
||||||
|
public void handleNullField(
|
||||||
|
boolean allowNullChkBox,
|
||||||
|
String defaultValue,
|
||||||
|
boolean nullValueNull,
|
||||||
|
String col,
|
||||||
|
StringBuilder rowValue,
|
||||||
|
boolean quote
|
||||||
|
) {
|
||||||
|
|
||||||
|
if(allowNullChkBox) {//cell nullable
|
||||||
|
if(defaultValue != null && !defaultValue.isEmpty()) {
|
||||||
|
if(quote) {
|
||||||
|
rowValue.append("'" + defaultValue + "'");
|
||||||
|
}else {
|
||||||
|
rowValue.append(defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
}else {
|
||||||
|
if(nullValueNull) {
|
||||||
|
rowValue.append("null");
|
||||||
|
|
||||||
|
}else {
|
||||||
|
throw new SqlExporterException("Null value not allowed for Field :" + col);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}else {
|
||||||
|
if(defaultValue != null && !defaultValue.isEmpty()) {
|
||||||
|
if(quote) {
|
||||||
|
rowValue.append("'" + defaultValue + "'");
|
||||||
|
}else {
|
||||||
|
rowValue.append(defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
}else {
|
||||||
|
throw new SqlExporterException("Null value not allowed for Field :" + col);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -41,6 +41,8 @@ import java.io.IOException;
|
|||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.json.JSONArray;
|
import org.json.JSONArray;
|
||||||
@ -212,19 +214,23 @@ public class SqlExporterTests extends RefineTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
String result = writer.toString();
|
String result = writer.toString();
|
||||||
|
//logger.info("result = " + result);
|
||||||
|
|
||||||
Assert.assertNotNull(result);
|
Assert.assertNotNull(result);
|
||||||
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
|
// assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
|
||||||
assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR);
|
// assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR);
|
||||||
|
|
||||||
boolean checkResult = result.contains("CREATE TABLE " + tableName);
|
boolean checkResult = result.contains("CREATE TABLE " + tableName);
|
||||||
Assert.assertEquals(checkResult, true);
|
Assert.assertEquals(checkResult, true);
|
||||||
|
//logger.info("checkResult1 = " + checkResult);
|
||||||
|
|
||||||
checkResult = result.contains("INSERT INTO " + tableName );
|
checkResult = result.contains("INSERT INTO " + tableName );
|
||||||
Assert.assertEquals(checkResult, true);
|
Assert.assertEquals(checkResult, true);
|
||||||
|
//logger.info("checkResult2 = " + checkResult);
|
||||||
|
|
||||||
checkResult = result.contains("DROP TABLE " + tableName + ";");
|
checkResult = result.contains("DROP TABLE IF EXISTS " + tableName + ";");
|
||||||
Assert.assertEquals(checkResult, true);
|
Assert.assertEquals(checkResult, true);
|
||||||
|
//logger.info("checkResult3 = " + checkResult);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -246,8 +252,82 @@ public class SqlExporterTests extends RefineTest {
|
|||||||
Assert.assertEquals(result, true);
|
Assert.assertEquals(result, true);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExportSqlWithNullFields(){
|
||||||
|
int inNull = 8;
|
||||||
|
createGridWithNullFields(3, 3, inNull);
|
||||||
|
String tableName = "sql_table_test";
|
||||||
|
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
|
||||||
|
optionsJson.put("includeStructure", true);
|
||||||
|
optionsJson.put("includeDropStatement", true);
|
||||||
|
optionsJson.put("convertNulltoEmptyString", true);
|
||||||
|
|
||||||
|
|
||||||
|
when(options.getProperty("options")).thenReturn(optionsJson.toString());
|
||||||
|
//logger.info("Options = " + optionsJson.toString());
|
||||||
|
|
||||||
|
try {
|
||||||
|
SUT.export(project, options, engine, writer);
|
||||||
|
} catch (IOException e) {
|
||||||
|
Assert.fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
String result = writer.toString();
|
||||||
|
Assert.assertNotNull(result);
|
||||||
|
//logger.info("\nresult = " + result);
|
||||||
|
// logger.info("\nNull Count:" + countWordInString(result, "null"));
|
||||||
|
|
||||||
|
int countNull = countWordInString(result, "null");
|
||||||
|
Assert.assertEquals(countNull, inNull);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExportSqlWithNotNullColumns(){
|
||||||
|
int noOfCols = 4;
|
||||||
|
int noOfRows = 3;
|
||||||
|
createGrid(noOfRows, noOfCols);
|
||||||
|
String tableName = "sql_table_test";
|
||||||
|
JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false);
|
||||||
|
optionsJson.put("includeStructure", true);
|
||||||
|
optionsJson.put("includeDropStatement", true);
|
||||||
|
optionsJson.put("convertNulltoEmptyString", true);
|
||||||
|
|
||||||
|
when(options.getProperty("options")).thenReturn(optionsJson.toString());
|
||||||
|
try {
|
||||||
|
SUT.export(project, options, engine, writer);
|
||||||
|
} catch (IOException e) {
|
||||||
|
Assert.fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
String result = writer.toString();
|
||||||
|
logger.info("\nresult:={} ", result);
|
||||||
|
Assert.assertNotNull(result);
|
||||||
|
|
||||||
|
int countNull = countWordInString(result, "NOT NULL");
|
||||||
|
logger.info("\nNot Null Count: {}" , countNull);
|
||||||
|
Assert.assertEquals(countNull, noOfCols);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
//helper methods
|
//helper methods
|
||||||
|
|
||||||
|
public int countWordInString(String input, String word){
|
||||||
|
if(input == null || input.isEmpty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int i = 0;
|
||||||
|
Pattern p = Pattern.compile(word);
|
||||||
|
Matcher m = p.matcher( input );
|
||||||
|
while (m.find()) {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return i;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
protected void createColumns(int noOfColumns){
|
protected void createColumns(int noOfColumns){
|
||||||
for(int i = 0; i < noOfColumns; i++){
|
for(int i = 0; i < noOfColumns; i++){
|
||||||
try {
|
try {
|
||||||
@ -269,6 +349,27 @@ public class SqlExporterTests extends RefineTest {
|
|||||||
project.rows.add(row);
|
project.rows.add(row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected void createGridWithNullFields(int noOfRows, int noOfColumns, int noOfNullFields){
|
||||||
|
createColumns(noOfColumns);
|
||||||
|
if(noOfNullFields > (noOfColumns * noOfRows)) {
|
||||||
|
noOfNullFields = noOfColumns * noOfRows;
|
||||||
|
}
|
||||||
|
int k = 0;
|
||||||
|
for(int i = 0; i < noOfRows; i++){
|
||||||
|
Row row = new Row(noOfColumns);
|
||||||
|
for(int j = 0; j < noOfColumns; j++){
|
||||||
|
if(k < noOfNullFields) {
|
||||||
|
row.cells.add(new Cell("", null));
|
||||||
|
k++;
|
||||||
|
}else {
|
||||||
|
row.cells.add(new Cell("row" + i + "cell" + j, null));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
project.rows.add(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected JSONObject createOptionsFromProject(String tableName, String type, String size) {
|
protected JSONObject createOptionsFromProject(String tableName, String type, String size) {
|
||||||
|
|
||||||
@ -283,8 +384,17 @@ public class SqlExporterTests extends RefineTest {
|
|||||||
//logger.info("Column Name = " + c.getName());
|
//logger.info("Column Name = " + c.getName());
|
||||||
JSONObject columnModel = new JSONObject();
|
JSONObject columnModel = new JSONObject();
|
||||||
columnModel.put("name", c.getName());
|
columnModel.put("name", c.getName());
|
||||||
columnModel.put("type", "VARCHAR");
|
if(type != null) {
|
||||||
columnModel.put("size", "100");
|
columnModel.put("type", type);
|
||||||
|
}else {
|
||||||
|
columnModel.put("type", "VARCHAR");
|
||||||
|
}
|
||||||
|
if(size != null) {
|
||||||
|
columnModel.put("size", size);
|
||||||
|
}else {
|
||||||
|
columnModel.put("size", "100");
|
||||||
|
}
|
||||||
|
|
||||||
if(type != null) {
|
if(type != null) {
|
||||||
columnModel.put("type", type);
|
columnModel.put("type", type);
|
||||||
}
|
}
|
||||||
@ -300,5 +410,48 @@ public class SqlExporterTests extends RefineTest {
|
|||||||
return json;
|
return json;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue,
|
||||||
|
boolean allowNull) {
|
||||||
|
|
||||||
|
JSONObject json = new JSONObject();
|
||||||
|
JSONArray columns = new JSONArray();
|
||||||
|
json.put("columns", columns);
|
||||||
|
json.put("tableName", tableName);
|
||||||
|
|
||||||
|
List<Column> cols = project.columnModel.columns;
|
||||||
|
|
||||||
|
cols.forEach(c -> {
|
||||||
|
//logger.info("Column Name = " + c.getName());
|
||||||
|
JSONObject columnModel = new JSONObject();
|
||||||
|
columnModel.put("name", c.getName());
|
||||||
|
if(type != null) {
|
||||||
|
columnModel.put("type", type);
|
||||||
|
}else {
|
||||||
|
columnModel.put("type", "VARCHAR");
|
||||||
|
}
|
||||||
|
if(size != null) {
|
||||||
|
columnModel.put("size", size);
|
||||||
|
}else {
|
||||||
|
columnModel.put("size", "100");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(type != null) {
|
||||||
|
columnModel.put("type", type);
|
||||||
|
}
|
||||||
|
if(size != null) {
|
||||||
|
// logger.info(" Size = " + size);
|
||||||
|
columnModel.put("size", size);
|
||||||
|
}
|
||||||
|
|
||||||
|
columnModel.put("defaultValue", defaultValue);
|
||||||
|
columnModel.put("allowNull", allowNull);
|
||||||
|
|
||||||
|
columns.put(columnModel);
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -151,8 +151,7 @@ function registerCommands() {
|
|||||||
|
|
||||||
RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand());
|
RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand());
|
||||||
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand());
|
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand());
|
||||||
|
|
||||||
RS.registerCommand(module, "preview-sql-export", new Packages.com.google.refine.commands.exporting.SqlExporterCommand());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function registerOperations() {
|
function registerOperations() {
|
||||||
|
@ -336,69 +336,43 @@ function SqlExporterDialog(options) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
|
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
|
||||||
var sqlExportInput = {};
|
|
||||||
sqlExportInput.name = name;
|
var format = options.format;
|
||||||
sqlExportInput.options = JSON.stringify(options);
|
var encoding = options.encoding;
|
||||||
sqlExportInput.project = theProject.id;
|
|
||||||
|
|
||||||
//validate form @ backend
|
delete options.format;
|
||||||
$.post(
|
delete options.encoding;
|
||||||
"command/core/preview-sql-export",
|
if (preview) {
|
||||||
sqlExportInput,
|
options.limit = 10;
|
||||||
function(sqlValidationResult) {
|
}
|
||||||
if(sqlValidationResult && sqlValidationResult.code === 'OK'){
|
|
||||||
//generate code
|
|
||||||
|
|
||||||
var format = options.format;
|
|
||||||
var encoding = options.encoding;
|
|
||||||
|
|
||||||
delete options.format;
|
|
||||||
delete options.encoding;
|
|
||||||
if (preview) {
|
|
||||||
options.limit = 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// var ext = SqlExporterDialog.formats[format].extension;
|
|
||||||
var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql");
|
|
||||||
$('<input />')
|
|
||||||
.attr("name", "options")
|
|
||||||
.attr("value", JSON.stringify(options))
|
|
||||||
.appendTo(form);
|
|
||||||
if (encoding) {
|
|
||||||
$('<input />')
|
|
||||||
.attr("name", "encoding")
|
|
||||||
.attr("value", encoding)
|
|
||||||
.appendTo(form);
|
|
||||||
}
|
|
||||||
if (!preview) {
|
|
||||||
$('<input />')
|
|
||||||
.attr("name", "contentType")
|
|
||||||
.attr("value", "application/x-unknown") // force download
|
|
||||||
.appendTo(form);
|
|
||||||
}
|
|
||||||
|
|
||||||
// alert("form::" + form);
|
|
||||||
document.body.appendChild(form);
|
|
||||||
|
|
||||||
window.open("about:blank", "refine-export");
|
|
||||||
form.submit();
|
|
||||||
|
|
||||||
document.body.removeChild(form);
|
|
||||||
return true;
|
|
||||||
|
|
||||||
}else{
|
|
||||||
window.alert("Problem with sql code genration");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
},
|
|
||||||
"json"
|
|
||||||
).fail(function( jqXhr, textStatus, errorThrown ){
|
|
||||||
alert( textStatus + ':' + errorThrown );
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
|
|
||||||
return true;
|
// var ext = SqlExporterDialog.formats[format].extension;
|
||||||
|
var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql");
|
||||||
|
$('<input />')
|
||||||
|
.attr("name", "options")
|
||||||
|
.attr("value", JSON.stringify(options))
|
||||||
|
.appendTo(form);
|
||||||
|
if (encoding) {
|
||||||
|
$('<input />')
|
||||||
|
.attr("name", "encoding")
|
||||||
|
.attr("value", encoding)
|
||||||
|
.appendTo(form);
|
||||||
|
}
|
||||||
|
if (!preview) {
|
||||||
|
$('<input />')
|
||||||
|
.attr("name", "contentType")
|
||||||
|
.attr("value", "application/x-unknown") // force download
|
||||||
|
.appendTo(form);
|
||||||
|
}
|
||||||
|
|
||||||
|
// alert("form::" + form);
|
||||||
|
document.body.appendChild(form);
|
||||||
|
|
||||||
|
window.open("about:blank", "refine-export");
|
||||||
|
form.submit();
|
||||||
|
|
||||||
|
document.body.removeChild(form);
|
||||||
|
return true;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user