s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added null and default value options

Browse Source
This commit is contained in:
TonyO 2018-04-26 20:50:46 -05:00
parent c7c0d8884a
commit 0eb7b4082f
12 changed files with 348 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
main/src/com/google/refine/commands/exporting/SqlExporterCommand.java
View File

@ -1,88 +0,0 @@
package com.google.refine.commands.exporting;
import java.io.IOException;
import java.util.Enumeration;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.exporters.sql.SqlDataError;
import com.google.refine.exporters.sql.SqlExporter;
import com.google.refine.model.Project;
public class SqlExporterCommand extends Command {
private static final Logger logger = LoggerFactory.getLogger("SqlExporterCommand");
@SuppressWarnings("unchecked")
public Properties getRequestParameters(HttpServletRequest request) {
Properties options = new Properties();
Enumeration<String> en = request.getParameterNames();
while (en.hasMoreElements()) {
String name = en.nextElement();
String value = request.getParameter(name);
options.put(name, value);
}
return options;
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
Project project = getProject(request);
//Engine engine = getEngine(request, project);
Properties params = getRequestParameters(request);
SqlExporter sqlExporter = new SqlExporter();
List<SqlDataError> result = sqlExporter.validateSqlData(project, params);
if(result == null || result.isEmpty()) {
writer.key("code");
writer.value("OK");
}else {
writer.key("code");
writer.value("Error");
writer.key("SqlDataErrors");
writer.array();
result.stream().forEach(err -> {
writer.object();
writer.key("errorcode");
writer.value(err.getErrorCode());
writer.key("errormessage");
writer.value(err.getErrorMessage());
writer.endObject();
});
writer.endArray();
}
writer.endObject();
} catch (Exception e) {
throw new ServletException(e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

16
main/src/com/google/refine/commands/project/ExportRowsCommand.java
View File

@ -44,6 +44,10 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
@ -52,9 +56,12 @@ import com.google.refine.exporters.Exporter;
import com.google.refine.exporters.ExporterRegistry;
import com.google.refine.exporters.StreamExporter;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.exporters.sql.SqlExporterException;
import com.google.refine.model.Project;
public class ExportRowsCommand extends Command {
private static final Logger logger = LoggerFactory.getLogger("ExportRowsCommand");
@SuppressWarnings("unchecked")
static public Properties getRequestParameters(HttpServletRequest request) {
@ -73,6 +80,7 @@ public class ExportRowsCommand extends Command {
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
@ -100,7 +108,8 @@ public class ExportRowsCommand extends Command {
((WriterExporter) exporter).export(project, params, engine, writer);
writer.close();
} else if (exporter instanceof StreamExporter) {
}
else if (exporter instanceof StreamExporter) {
response.setCharacterEncoding("UTF-8");
OutputStream stream = response.getOutputStream();
@ -108,12 +117,17 @@ public class ExportRowsCommand extends Command {
stream.close();
// } else if (exporter instanceof UrlExporter) {
// ((UrlExporter) exporter).export(project, options, engine);
} else {
// TODO: Should this use ServletException instead of respondException?
respondException(response, new RuntimeException("Unknown exporter type"));
}
} catch (Exception e) {
// Use generic error handling rather than our JSON handling
logger.info("error:{}", e.getMessage());
if (e instanceof SqlExporterException) {
response.sendError(HttpStatus.SC_BAD_REQUEST, e.getMessage());
}
throw new ServletException(e);
} finally {
ProjectManager.singleton.setBusy(false);

12
main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java
View File

@ -229,12 +229,16 @@ abstract public class CustomizableTabularExporterUtilities {
Map<String, String> identifierSpaceToUrl = null;
//SQLExporter parameter to convert null cell value to empty string
boolean includeNullFieldValue = false;
CellFormatter() {
dateFormatter = new SimpleDateFormat(fullIso8601);
}
CellFormatter(JSONObject options) {
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings");
includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false);
if (reconSettings != null) {
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null);
if ("entity-name".equals(reconOutputString)) {
@ -354,6 +358,12 @@ abstract public class CustomizableTabularExporterUtilities {
}
return new CellData(column.getName(), value, text, link);
}
}else {//added for sql exporter
if(includeNullFieldValue) {
return new CellData(column.getName(), "", "", "");
}
}
return null;
}
@ -384,4 +394,4 @@ abstract public class CustomizableTabularExporterUtilities {
}
}
}
}
}

11
main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java
View File

@ -43,16 +43,15 @@ public class SqlCreateBuilder {
private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder");
private String table;
private List<String> columns;
private JSONObject options;
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) {
this.table = table;
this.columns = columns;
this.options = options;
}
public String getCreateSQL() {
@ -76,6 +75,7 @@ public class SqlCreateBuilder {
String size = JSONUtilities.getString(columnOptions, "size", "");
boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true);
String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null);
logger.info("allowNull::{}" , allowNull);
String allowNullStr = "NULL";
if(!allowNull) {
@ -125,6 +125,11 @@ public class SqlCreateBuilder {
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
createSB.append(" DEFAULT " + "'" + defaultValue + "'");
}else {
try {
Integer.parseInt(defaultValue);
}catch(NumberFormatException nfe) {
throw new SqlExporterException(defaultValue + " is not compatible with column type :" + type);
}
createSB.append(" DEFAULT " + defaultValue);
}

2
main/src/com/google/refine/exporters/sql/SqlData.java
View File

@ -41,6 +41,8 @@ public class SqlData {
public static final String SQL_TYPE_INTEGER = "INTEGER";
public static final String SQL_TYPE_INT = "INT";
public static final String SQL_TYPE_NUMERIC = "NUMERIC";
public static final String SQL_TYPE_DATE = "DATE";
public static final String SQL_TYPE_TIMESTAMP = "TIMESTAMP";

34
main/src/com/google/refine/exporters/sql/SqlDataError.java
View File

@ -1,34 +0,0 @@
package com.google.refine.exporters.sql;
public class SqlDataError {
private int errorCode;
private String errorMessage;
public SqlDataError(int errorCode, String errorMessage) {
super();
this.errorCode = errorCode;
this.errorMessage = errorMessage;
}
public int getErrorCode() {
return errorCode;
}
public void setErrorCode(int errorCode) {
this.errorCode = errorCode;
}
public String getErrorMessage() {
return errorMessage;
}
public void setErrorMessage(String errorMessage) {
this.errorMessage = errorMessage;
}
}

19
main/src/com/google/refine/exporters/sql/SqlExporter.java
View File

@ -60,6 +60,7 @@ public class SqlExporter implements WriterExporter {
private List<String> columnNames = new ArrayList<String>();
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
private JSONObject sqlOptions;
@Override
public String getContentType() {
@ -72,6 +73,7 @@ public class SqlExporter implements WriterExporter {
if(logger.isDebugEnabled()) {
logger.debug("export sql with params: {}", params);
}
TabularSerializer serializer = new TabularSerializer() {
@Override
@ -84,14 +86,14 @@ public class SqlExporter implements WriterExporter {
public void endFile() {
try {
if (columnNames.isEmpty()) {
writer.write(NO_COL_SELECTED_ERROR);
logger.error("No Columns Selected!!");
return;
throw new SqlExporterException(NO_COL_SELECTED_ERROR);
}
if (sqlOptions == null) {
writer.write(NO_OPTIONS_PRESENT_ERROR);
logger.error("No Options Selected!!");
return;
throw new SqlExporterException(NO_OPTIONS_PRESENT_ERROR);
}
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
@ -163,11 +165,6 @@ public class SqlExporter implements WriterExporter {
CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer);
}
public List<SqlDataError> validateSqlData(Project project, Properties params){
logger.info("Param Name:{}, Param Value:{}", project, params);
return null;
}
}

16
main/src/com/google/refine/exporters/sql/SqlExporterException.java Normal file
View File

@ -0,0 +1,16 @@
package com.google.refine.exporters.sql;
public class SqlExporterException extends RuntimeException {
/**
*
*/
private static final long serialVersionUID = -2640105469265805010L;
public SqlExporterException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
}

120
main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java
View File

@ -53,6 +53,7 @@ public class SqlInsertBuilder {
private List<ArrayList<SqlData>> sqlDataList;
private JSONObject options;
/**
*
@ -60,12 +61,16 @@ public class SqlInsertBuilder {
* @param columns
* @param rows
* @param options
* @param sqlErrors
*/
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options) {
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options
) {
this.table = table;
this.columns = columns;
this.sqlDataList = rows;
this.options = options;
//logger.info("Column Size:{}", columns.size());
}
/**
@ -86,65 +91,74 @@ public class SqlInsertBuilder {
});
}
boolean nullValueToEmptyStr = options == null ? false : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true);
boolean nullValueNull = options == null ? true : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true);
StringBuffer values = new StringBuffer();
int idx = 0;
for(ArrayList<SqlData> sqlRow : sqlDataList) {
StringBuilder rowValue = new StringBuilder();
//int fieldCount = 0;
for(SqlData val : sqlRow) {
JSONObject jsonOb = colOptionsMap.get(val.getColumnName());
String type = (String)jsonOb.get("type");
String defaultValue = (String)jsonOb.get("defaultValue");
boolean allowNullChkBox = (boolean)jsonOb.get("allowNull");
String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null);
boolean allowNullChkBox = JSONUtilities.getBoolean(jsonOb, "defaultValue", true);;
if(type == null) {
type = SqlData.SQL_TYPE_VARCHAR;
}
//Character Types
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
if(!allowNullChkBox) {
throw new RuntimeException("bad input");
}
if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) {
// logger.info("Appending empty String:::{}" , val.getText());
if(defaultValue != null && !defaultValue.isEmpty()) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append("null");
}
if((val.getText() == null || val.getText().isEmpty()) ) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
}else {
rowValue.append("'" + val.getText() + "'");
}
}else {//Numeric Types
}else if(type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER) || type.equals(SqlData.SQL_TYPE_NUMERIC)) {//Numeric Types : INT, NUMERIC
if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) {
// logger.info("Appending empty String others:::{}" , val.getText());
if(defaultValue != null && !defaultValue.isEmpty()) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append("null");
if((val.getText() == null || val.getText().isEmpty())) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, false);
}else {//value not null
try {
Integer.parseInt(val.getText());
} catch (NumberFormatException nfe) {
throw new SqlExporterException(
val.getText() + " is not compatible with column type :" + type);
}
}else {
rowValue.append(val.getText());
}
}else if(type.equals(SqlData.SQL_TYPE_DATE) || type.equals(SqlData.SQL_TYPE_TIMESTAMP)) {
if((val.getText() == null || val.getText().isEmpty())) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
}else {
rowValue.append("'" + val.getText() + "'");
}
}
rowValue.append(",");
}
idx++;
String rowValString = rowValue.toString();
// logger.info("rowValString::" + rowValString);
rowValString = rowValString.substring(0, rowValString.length() - 1);
values.append("( ");
@ -182,4 +196,58 @@ public class SqlInsertBuilder {
return sqlString;
}
/**
*
* @param allowNullChkBox
* @param defaultValue
* @param nullValueNull
* @param col
* @param rowValue
* @param quote
* @param fieldCount
*/
public void handleNullField(
boolean allowNullChkBox,
String defaultValue,
boolean nullValueNull,
String col,
StringBuilder rowValue,
boolean quote
) {
if(allowNullChkBox) {//cell nullable
if(defaultValue != null && !defaultValue.isEmpty()) {
if(quote) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append(defaultValue);
}
}else {
if(nullValueNull) {
rowValue.append("null");
}else {
throw new SqlExporterException("Null value not allowed for Field :" + col);
}
}
}else {
if(defaultValue != null && !defaultValue.isEmpty()) {
if(quote) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append(defaultValue);
}
}else {
throw new SqlExporterException("Null value not allowed for Field :" + col);
}
}
}
}

163
main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java
View File

@ -41,6 +41,8 @@ import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.json.JSONArray;
@ -212,19 +214,23 @@ public class SqlExporterTests extends RefineTest {
}
String result = writer.toString();
//logger.info("result = " + result);
Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR);
// assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
// assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
Assert.assertEquals(checkResult, true);
//logger.info("checkResult1 = " + checkResult);
checkResult = result.contains("INSERT INTO " + tableName );
Assert.assertEquals(checkResult, true);
//logger.info("checkResult2 = " + checkResult);
checkResult = result.contains("DROP TABLE " + tableName + ";");
checkResult = result.contains("DROP TABLE IF EXISTS " + tableName + ";");
Assert.assertEquals(checkResult, true);
//logger.info("checkResult3 = " + checkResult);
}
@ -246,8 +252,82 @@ public class SqlExporterTests extends RefineTest {
Assert.assertEquals(result, true);
}
@Test
public void testExportSqlWithNullFields(){
int inNull = 8;
createGridWithNullFields(3, 3, inNull);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
Assert.assertNotNull(result);
//logger.info("\nresult = " + result);
// logger.info("\nNull Count:" + countWordInString(result, "null"));
int countNull = countWordInString(result, "null");
Assert.assertEquals(countNull, inNull);
}
@Test
public void testExportSqlWithNotNullColumns(){
int noOfCols = 4;
int noOfRows = 3;
createGrid(noOfRows, noOfCols);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
logger.info("\nresult:={} ", result);
Assert.assertNotNull(result);
int countNull = countWordInString(result, "NOT NULL");
logger.info("\nNot Null Count: {}" , countNull);
Assert.assertEquals(countNull, noOfCols);
}
//helper methods
public int countWordInString(String input, String word){
if(input == null || input.isEmpty()) {
return 0;
}
int i = 0;
Pattern p = Pattern.compile(word);
Matcher m = p.matcher( input );
while (m.find()) {
i++;
}
return i;
}
protected void createColumns(int noOfColumns){
for(int i = 0; i < noOfColumns; i++){
try {
@ -269,6 +349,27 @@ public class SqlExporterTests extends RefineTest {
project.rows.add(row);
}
}
protected void createGridWithNullFields(int noOfRows, int noOfColumns, int noOfNullFields){
createColumns(noOfColumns);
if(noOfNullFields > (noOfColumns * noOfRows)) {
noOfNullFields = noOfColumns * noOfRows;
}
int k = 0;
for(int i = 0; i < noOfRows; i++){
Row row = new Row(noOfColumns);
for(int j = 0; j < noOfColumns; j++){
if(k < noOfNullFields) {
row.cells.add(new Cell("", null));
k++;
}else {
row.cells.add(new Cell("row" + i + "cell" + j, null));
}
}
project.rows.add(row);
}
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size) {
@ -283,8 +384,17 @@ public class SqlExporterTests extends RefineTest {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
columnModel.put("name", c.getName());
columnModel.put("type", "VARCHAR");
columnModel.put("size", "100");
if(type != null) {
columnModel.put("type", type);
}else {
columnModel.put("type", "VARCHAR");
}
if(size != null) {
columnModel.put("size", size);
}else {
columnModel.put("size", "100");
}
if(type != null) {
columnModel.put("type", type);
}
@ -300,5 +410,48 @@ public class SqlExporterTests extends RefineTest {
return json;
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue,
boolean allowNull) {
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
json.put("tableName", tableName);
List<Column> cols = project.columnModel.columns;
cols.forEach(c -> {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
columnModel.put("name", c.getName());
if(type != null) {
columnModel.put("type", type);
}else {
columnModel.put("type", "VARCHAR");
}
if(size != null) {
columnModel.put("size", size);
}else {
columnModel.put("size", "100");
}
if(type != null) {
columnModel.put("type", type);
}
if(size != null) {
// logger.info(" Size = " + size);
columnModel.put("size", size);
}
columnModel.put("defaultValue", defaultValue);
columnModel.put("allowNull", allowNull);
columns.put(columnModel);
});
return json;
}
}

3
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -151,8 +151,7 @@ function registerCommands() {
RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand());
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand());
RS.registerCommand(module, "preview-sql-export", new Packages.com.google.refine.commands.exporting.SqlExporterCommand());
}
function registerOperations() {

96
main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js
View File

@ -336,69 +336,43 @@ function SqlExporterDialog(options) {
}
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
var sqlExportInput = {};
sqlExportInput.name = name;
sqlExportInput.options = JSON.stringify(options);
sqlExportInput.project = theProject.id;
var format = options.format;
var encoding = options.encoding;
//validate form @ backend
$.post(
"command/core/preview-sql-export",
sqlExportInput,
function(sqlValidationResult) {
if(sqlValidationResult && sqlValidationResult.code === 'OK'){
//generate code
var format = options.format;
var encoding = options.encoding;
delete options.format;
delete options.encoding;
if (preview) {
options.limit = 10;
}
// var ext = SqlExporterDialog.formats[format].extension;
var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql");
$('<input />')
.attr("name", "options")
.attr("value", JSON.stringify(options))
.appendTo(form);
if (encoding) {
$('<input />')
.attr("name", "encoding")
.attr("value", encoding)
.appendTo(form);
}
if (!preview) {
$('<input />')
.attr("name", "contentType")
.attr("value", "application/x-unknown") // force download
.appendTo(form);
}
// alert("form::" + form);
document.body.appendChild(form);
window.open("about:blank", "refine-export");
form.submit();
document.body.removeChild(form);
return true;
}else{
window.alert("Problem with sql code genration");
return false;
}
},
"json"
).fail(function( jqXhr, textStatus, errorThrown ){
alert( textStatus + ':' + errorThrown );
return false;
});
delete options.format;
delete options.encoding;
if (preview) {
options.limit = 10;
}
return true;
// var ext = SqlExporterDialog.formats[format].extension;
var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql");
$('<input />')
.attr("name", "options")
.attr("value", JSON.stringify(options))
.appendTo(form);
if (encoding) {
$('<input />')
.attr("name", "encoding")
.attr("value", encoding)
.appendTo(form);
}
if (!preview) {
$('<input />')
.attr("name", "contentType")
.attr("value", "application/x-unknown") // force download
.appendTo(form);
}
// alert("form::" + form);
document.body.appendChild(form);
window.open("about:blank", "refine-export");
form.submit();
document.body.removeChild(form);
return true;
};