s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

merge the translation

Browse Source
This commit is contained in:
Jacky 2018-05-16 15:40:05 -04:00
commit a103e1fb90
15 changed files with 1926 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
main/src/com/google/refine/commands/project/ExportRowsCommand.java
View File

@ -44,6 +44,10 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
@ -52,9 +56,12 @@ import com.google.refine.exporters.Exporter;
import com.google.refine.exporters.ExporterRegistry;
import com.google.refine.exporters.StreamExporter;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.exporters.sql.SqlExporterException;
import com.google.refine.model.Project;
public class ExportRowsCommand extends Command {
private static final Logger logger = LoggerFactory.getLogger("ExportRowsCommand");
@SuppressWarnings("unchecked")
static public Properties getRequestParameters(HttpServletRequest request) {
@ -73,6 +80,7 @@ public class ExportRowsCommand extends Command {
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
@ -100,7 +108,8 @@ public class ExportRowsCommand extends Command {
((WriterExporter) exporter).export(project, params, engine, writer);
writer.close();
} else if (exporter instanceof StreamExporter) {
}
else if (exporter instanceof StreamExporter) {
response.setCharacterEncoding("UTF-8");
OutputStream stream = response.getOutputStream();
@ -108,12 +117,17 @@ public class ExportRowsCommand extends Command {
stream.close();
// } else if (exporter instanceof UrlExporter) {
// ((UrlExporter) exporter).export(project, options, engine);
} else {
// TODO: Should this use ServletException instead of respondException?
respondException(response, new RuntimeException("Unknown exporter type"));
}
} catch (Exception e) {
// Use generic error handling rather than our JSON handling
logger.info("error:{}", e.getMessage());
if (e instanceof SqlExporterException) {
response.sendError(HttpStatus.SC_BAD_REQUEST, e.getMessage());
}
throw new ServletException(e);
} finally {
ProjectManager.singleton.setBusy(false);

12
main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java
View File

@ -229,12 +229,16 @@ abstract public class CustomizableTabularExporterUtilities {
Map<String, String> identifierSpaceToUrl = null;
//SQLExporter parameter to convert null cell value to empty string
boolean includeNullFieldValue = false;
CellFormatter() {
dateFormatter = new SimpleDateFormat(fullIso8601);
}
CellFormatter(JSONObject options) {
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings");
includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false);
if (reconSettings != null) {
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null);
if ("entity-name".equals(reconOutputString)) {
@ -354,6 +358,12 @@ abstract public class CustomizableTabularExporterUtilities {
}
return new CellData(column.getName(), value, text, link);
}
}else {//added for sql exporter
if(includeNullFieldValue) {
return new CellData(column.getName(), "", "", "");
}
}
return null;
}
@ -384,4 +394,4 @@ abstract public class CustomizableTabularExporterUtilities {
}
}
}
}
}

4
main/src/com/google/refine/exporters/ExporterRegistry.java
View File

@ -36,6 +36,8 @@ package com.google.refine.exporters;
import java.util.HashMap;
import java.util.Map;
import com.google.refine.exporters.sql.SqlExporter;
abstract public class ExporterRegistry {
static final private Map<String, Exporter> s_formatToExporter = new HashMap<String, Exporter>();
@ -52,6 +54,8 @@ abstract public class ExporterRegistry {
s_formatToExporter.put("html", new HtmlTableExporter());
s_formatToExporter.put("template", new TemplatingExporter());
s_formatToExporter.put("sql", new SqlExporter());
}
static public void registerExporter(String format, Exporter exporter) {

173
main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java Executable file
View File

@ -0,0 +1,173 @@
/*
* Copyright (c) 2018, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.exporters.sql;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.util.JSONUtilities;
public class SqlCreateBuilder {
private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder");
private String table;
private List<String> columns;
private JSONObject options;
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) {
this.table = table;
this.columns = columns;
this.options = options;
}
public String getCreateSQL() {
if(logger.isDebugEnabled()) {
logger.debug("Create SQL with columns: {}", columns);
}
StringBuffer createSB = new StringBuffer();
JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false);
int count = columnOptionArray.length();
for (int i = 0; i < count; i++) {
JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i);
if (columnOptions != null) {
String name = JSONUtilities.getString(columnOptions, "name", null);
String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR);
String size = JSONUtilities.getString(columnOptions, "size", "");
boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true);
String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null);
logger.info("allowNull::{}" , allowNull);
String allowNullStr = "NULL";
if(!allowNull) {
allowNullStr = "NOT NULL";
}
if (name != null) {
if(trimColNames) {
String trimmedCol = name.replaceAll("\\s", "");
createSB.append( trimmedCol + " ");
}else{
createSB.append(name + " ");
}
if (type.equals(SqlData.SQL_TYPE_VARCHAR)) {
if (size.isEmpty()) {
size = "255";
}
createSB.append(type + "(" + size + ")");
} else if (type.equals(SqlData.SQL_TYPE_CHAR)) {
if (size.isEmpty()) {
size = "10";
}
createSB.append(type + "(" + size + ")");
} else if (type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER)) {
if (size.isEmpty()) {
createSB.append(type);
} else {
createSB.append(type + "(" + size + ")");
}
} else if (type.equals(SqlData.SQL_TYPE_NUMERIC)) {
if (size.isEmpty()) {
createSB.append(type);
} else {
createSB.append(type + "(" + size + ")");
}
} else {
createSB.append(type);
}
createSB.append(" " + allowNullStr);
if(defaultValue != null && !defaultValue.isEmpty()) {
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
createSB.append(" DEFAULT " + "'" + defaultValue + "'");
}else {
try {
Integer.parseInt(defaultValue);
}catch(NumberFormatException nfe) {
throw new SqlExporterException(defaultValue + " is not compatible with column type :" + type);
}
createSB.append(" DEFAULT " + defaultValue);
}
}
if (i < count - 1) {
createSB.append(",");
}
createSB.append("\n");
}
}
}
StringBuffer sql = new StringBuffer();
boolean includeDrop = JSONUtilities.getBoolean(options, "includeDropStatement", false);
boolean addIfExist = options == null ? false : JSONUtilities.getBoolean(options, "includeIfExistWithDropStatement", true);
if (includeDrop) {
if(addIfExist) {
sql.append("DROP TABLE IF EXISTS " + table + ";\n");
}else {
sql.append("DROP TABLE " + table + ";\n");
}
}
sql.append("CREATE TABLE ").append(table);
sql.append(" (").append("\n");
sql.append(createSB.toString());
sql.append(")").append(";" + "\n");
String createSQL = sql.toString();
if(logger.isDebugEnabled()){
logger.debug("Create SQL Generated Successfully...{}", createSQL);
}
return createSQL;
}
}

77
main/src/com/google/refine/exporters/sql/SqlData.java Executable file
View File

@ -0,0 +1,77 @@
/*
* Copyright (c) 2018, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.exporters.sql;
public class SqlData {
private String columnName;
private Object value;
private String text;
public static final String SQL_TYPE_VARCHAR = "VARCHAR";
public static final String SQL_TYPE_CHAR = "CHAR";
public static final String SQL_TYPE_TEXT = "TEXT";
public static final String SQL_TYPE_INTEGER = "INTEGER";
public static final String SQL_TYPE_INT = "INT";
public static final String SQL_TYPE_NUMERIC = "NUMERIC";
public static final String SQL_TYPE_DATE = "DATE";
public static final String SQL_TYPE_TIMESTAMP = "TIMESTAMP";
public SqlData(String columnName, Object value, String text) {
this.columnName = columnName;
this.value = value;
this.text = text;
}
public String getColumnName() {
return columnName;
}
public Object getValue() {
return value;
}
public String getText() {
return text;
}
@Override
public String toString() {
return "SqlData [columnName=" + columnName + ", value=" + value + ", text=" + text + "]";
}
}

170
main/src/com/google/refine/exporters/sql/SqlExporter.java Executable file
View File

@ -0,0 +1,170 @@
/*
* Copyright (c) 2018, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.exporters.sql;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.CustomizableTabularExporterUtilities;
import com.google.refine.exporters.TabularSerializer;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project;
import com.google.refine.util.JSONUtilities;
public class SqlExporter implements WriterExporter {
private static final Logger logger = LoggerFactory.getLogger("SqlExporter");
public static final String NO_COL_SELECTED_ERROR = "****NO COLUMNS SELECTED****";
public static final String NO_OPTIONS_PRESENT_ERROR = "****NO OPTIONS PRESENT****";
//JSON Property names
public static final String JSON_INCLUDE_STRUCTURE = "includeStructure";
public static final String JSON_INCLUDE_CONTENT = "includeContent";
public static final String JSON_TABLE_NAME = "tableName";
private List<String> columnNames = new ArrayList<String>();
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
private JSONObject sqlOptions;
@Override
public String getContentType() {
return "text/plain";
}
@Override
public void export(final Project project, Properties params, Engine engine, final Writer writer)
throws IOException {
if(logger.isDebugEnabled()) {
logger.debug("export sql with params: {}", params);
}
TabularSerializer serializer = new TabularSerializer() {
@Override
public void startFile(JSONObject options) {
sqlOptions = options;
//logger.info("setting options::{}", sqlOptions);
}
@Override
public void endFile() {
try {
if (columnNames.isEmpty()) {
logger.error("No Columns Selected!!");
throw new SqlExporterException(NO_COL_SELECTED_ERROR);
}
if (sqlOptions == null) {
logger.error("No Options Selected!!");
throw new SqlExporterException(NO_OPTIONS_PRESENT_ERROR);
}
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
Object tableNameManual = sqlOptions.get(JSON_TABLE_NAME);
if (tableNameManual != null && !tableNameManual.toString().isEmpty()) {
tableName = tableNameManual.toString();
}
SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions);
SqlInsertBuilder insertBuilder = new SqlInsertBuilder(tableName, columnNames, sqlDataList,
sqlOptions);
final boolean includeStructure = sqlOptions == null ? true
: JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_STRUCTURE, true);
final boolean includeContent = sqlOptions == null ? true
: JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_CONTENT, true);
if (includeStructure) {
String sqlCreateStr = createBuilder.getCreateSQL();
writer.write(sqlCreateStr);
}
if (includeContent) {
String sqlInsertStr = insertBuilder.getInsertSQL();
writer.write(sqlInsertStr);
}
if (logger.isDebugEnabled()) {
logger.debug("sqlOptions::{}", sqlOptions);
}
columnNames = new ArrayList<String>();
sqlDataList = new ArrayList<ArrayList<SqlData>>();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void addRow(List<CellData> cells, boolean isHeader) {
if (isHeader) {
for (CellData cellData : cells) {
columnNames.add(cellData.text);
}
} else {
ArrayList<SqlData> values = new ArrayList<>();
for (CellData cellData : cells) {
if (cellData != null) {
if(cellData.text == null || cellData.text.isEmpty()) {
values.add(new SqlData(cellData.columnName, "", ""));
}else {
values.add(new SqlData(cellData.columnName, cellData.value, cellData.text));
}
}
}
sqlDataList.add(values);
}
}
};
CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer);
}
}

16
main/src/com/google/refine/exporters/sql/SqlExporterException.java Normal file
View File

@ -0,0 +1,16 @@
package com.google.refine.exporters.sql;
public class SqlExporterException extends RuntimeException {
/**
*
*/
private static final long serialVersionUID = -2640105469265805010L;
public SqlExporterException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
}

253
main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java Executable file
View File

@ -0,0 +1,253 @@
/*
* Copyright (c) 2018, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.exporters.sql;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.util.JSONUtilities;
public class SqlInsertBuilder {
private static final Logger logger = LoggerFactory.getLogger("SQLInsertBuilder");
private String table;
private List<String> columns;
private List<ArrayList<SqlData>> sqlDataList;
private JSONObject options;
/**
*
* @param table
* @param columns
* @param rows
* @param options
* @param sqlErrors
*/
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options
) {
this.table = table;
this.columns = columns;
this.sqlDataList = rows;
this.options = options;
//logger.info("Column Size:{}", columns.size());
}
/**
* Get Insert Sql
* @return
*/
public String getInsertSQL(){
if(logger.isDebugEnabled()) {
logger.debug("Insert SQL with columns: {}", columns);
}
JSONArray colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
Map<String, JSONObject> colOptionsMap = new HashMap<String, JSONObject>();
if(colOptionArray != null) {
colOptionArray.forEach(c -> {
JSONObject json = (JSONObject)c;
colOptionsMap.put("" + json.get("name"), json);
});
}
boolean nullValueNull = options == null ? true : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true);
StringBuffer values = new StringBuffer();
int idx = 0;
for(ArrayList<SqlData> sqlRow : sqlDataList) {
StringBuilder rowValue = new StringBuilder();
//int fieldCount = 0;
for(SqlData val : sqlRow) {
JSONObject jsonOb = colOptionsMap.get(val.getColumnName());
String type = (String)jsonOb.get("type");
String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null);
boolean allowNullChkBox = JSONUtilities.getBoolean(jsonOb, "defaultValue", true);;
if(type == null) {
type = SqlData.SQL_TYPE_VARCHAR;
}
//Character Types
if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) {
if((val.getText() == null || val.getText().isEmpty()) ) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
}else {
rowValue.append("'" + val.getText() + "'");
}
}else if(type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER) || type.equals(SqlData.SQL_TYPE_NUMERIC)) {//Numeric Types : INT, NUMERIC
if((val.getText() == null || val.getText().isEmpty())) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, false);
}else {//value not null
try {
Integer.parseInt(val.getText());
} catch (NumberFormatException nfe) {
throw new SqlExporterException(
val.getText() + " is not compatible with column type :" + type);
}
rowValue.append(val.getText());
}
}else if(type.equals(SqlData.SQL_TYPE_DATE) || type.equals(SqlData.SQL_TYPE_TIMESTAMP)) {
if((val.getText() == null || val.getText().isEmpty())) {
handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true);
}else {
rowValue.append("'" + val.getText() + "'");
}
}
rowValue.append(",");
}
idx++;
String rowValString = rowValue.toString();
// logger.info("rowValString::" + rowValString);
rowValString = rowValString.substring(0, rowValString.length() - 1);
values.append("( ");
values.append(rowValString);
values.append(" )");
if(idx < sqlDataList.size()) {
values.append(",");
}
values.append("\n");
}
boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false);
String colNamesWithSep = columns.stream().map(col -> col.replaceAll("\\s", "")).collect(Collectors.joining(","));;
if(!trimColNames) {
colNamesWithSep = columns.stream().collect(Collectors.joining(","));
}
String valuesString = values.toString();
valuesString = valuesString.substring(0, valuesString.length() - 1);
StringBuffer sql = new StringBuffer();
sql.append("INSERT INTO ").append(table);
sql.append(" (");
sql.append(colNamesWithSep);
sql.append(") VALUES ").append("\n");
sql.append(valuesString);
String sqlString = sql.toString();
if(logger.isDebugEnabled()) {
logger.debug("Insert Statement Generated Successfully...{}", sqlString);
}
return sqlString;
}
/**
*
* @param allowNullChkBox
* @param defaultValue
* @param nullValueNull
* @param col
* @param rowValue
* @param quote
* @param fieldCount
*/
public void handleNullField(
boolean allowNullChkBox,
String defaultValue,
boolean nullValueNull,
String col,
StringBuilder rowValue,
boolean quote
) {
if(allowNullChkBox) {//cell nullable
if(defaultValue != null && !defaultValue.isEmpty()) {
if(quote) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append(defaultValue);
}
}else {
if(nullValueNull) {
rowValue.append("null");
}else {
throw new SqlExporterException("Null value not allowed for Field :" + col);
}
}
}else {
if(defaultValue != null && !defaultValue.isEmpty()) {
if(quote) {
rowValue.append("'" + defaultValue + "'");
}else {
rowValue.append(defaultValue);
}
}else {
throw new SqlExporterException("Null value not allowed for Field :" + col);
}
}
}
}

457
main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java Normal file
View File

@ -0,0 +1,457 @@
/*
Copyright 2018, Tony Opara.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.exporters.sql;
import static org.junit.Assert.assertNotEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.sql.SqlCreateBuilder;
import com.google.refine.exporters.sql.SqlExporter;
import com.google.refine.exporters.sql.SqlInsertBuilder;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.tests.ProjectManagerStub;
import com.google.refine.tests.RefineTest;
public class SqlExporterTests extends RefineTest {
private static final String TEST_PROJECT_NAME = "SQL_EXPORTER_TEST_PROJECT";
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
}
//dependencies
StringWriter writer;
ProjectMetadata projectMetadata;
Project project;
Engine engine;
Properties options;
SqlCreateBuilder sqlCreateBuilder;
SqlInsertBuilder sqlInsertBuilder;
//System Under Test
SqlExporter SUT;
@BeforeMethod
public void SetUp(){
SUT = new SqlExporter();
writer = new StringWriter();
ProjectManager.singleton = new ProjectManagerStub();
projectMetadata = new ProjectMetadata();
project = new Project();
projectMetadata.setName(TEST_PROJECT_NAME);
ProjectManager.singleton.registerProject(project, projectMetadata);
engine = new Engine(project);
options = mock(Properties.class);
}
@AfterMethod
public void TearDown(){
SUT = null;
writer = null;
ProjectManager.singleton.deleteProject(project.id);
project = null;
projectMetadata = null;
engine = null;
options = null;
sqlCreateBuilder = null;
sqlInsertBuilder = null;
}
@Test
public void testExportSimpleSql(){
createGrid(2, 2);
String tableName = "sql_table_test";
String optionsString = createOptionsFromProject(tableName, null,null).toString();
when(options.getProperty("options")).thenReturn(optionsString);
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
// logger.info("result = \n" + result);
Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
//logger.info("checkResult1 =" + checkResult);
checkResult = result.contains("INSERT INTO " + tableName);
// logger.info("checkResult2 =" + checkResult);
Assert.assertEquals(checkResult, true);
}
@Test
public void testExportSqlNoSchema(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null,null);
optionsJson.put("includeStructure", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
// logger.info("Options = " + optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
//logger.info("result = \n" + result);
Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
Assert.assertEquals(checkResult, false);
checkResult = result.contains("INSERT INTO " + tableName);
Assert.assertEquals(checkResult, true);
}
@Test
public void testExportSqlNoContent(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
optionsJson.put("includeContent", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
// logger.info("result = \n" + result);
Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
Assert.assertEquals(checkResult, true);
checkResult = result.contains("INSERT INTO " + tableName);
Assert.assertEquals(checkResult, false);
}
@Test
public void testExportSqlIncludeSchemaWithDropStmt(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
//logger.info("result = " + result);
Assert.assertNotNull(result);
// assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
// assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
Assert.assertEquals(checkResult, true);
//logger.info("checkResult1 = " + checkResult);
checkResult = result.contains("INSERT INTO " + tableName );
Assert.assertEquals(checkResult, true);
//logger.info("checkResult2 = " + checkResult);
checkResult = result.contains("DROP TABLE IF EXISTS " + tableName + ";");
Assert.assertEquals(checkResult, true);
//logger.info("checkResult3 = " + checkResult);
}
@Test
public void testGetCreateSql(){
createGrid(3,3);
String tableName = "sql_table_test";
String type = "CHAR";
String size = "2";
JSONObject optionsJson = createOptionsFromProject(tableName, type, size);
// logger.info("Options:: = " + optionsJson.toString());
List<String> columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList());
sqlCreateBuilder = new SqlCreateBuilder(tableName, columns, optionsJson);
String createSql = sqlCreateBuilder.getCreateSQL();
//logger.info("createSql = \n" + createSql);
Assert.assertNotNull(createSql);
boolean result = createSql.contains(type + "(" + size + ")");
Assert.assertEquals(result, true);
}
@Test
public void testExportSqlWithNullFields(){
int inNull = 8;
createGridWithNullFields(3, 3, inNull);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
Assert.assertNotNull(result);
//logger.info("\nresult = " + result);
// logger.info("\nNull Count:" + countWordInString(result, "null"));
int countNull = countWordInString(result, "null");
Assert.assertEquals(countNull, inNull);
}
@Test
public void testExportSqlWithNotNullColumns(){
int noOfCols = 4;
int noOfRows = 3;
createGrid(noOfRows, noOfCols);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
try {
SUT.export(project, options, engine, writer);
} catch (IOException e) {
Assert.fail();
}
String result = writer.toString();
logger.info("\nresult:={} ", result);
Assert.assertNotNull(result);
int countNull = countWordInString(result, "NOT NULL");
logger.info("\nNot Null Count: {}" , countNull);
Assert.assertEquals(countNull, noOfCols);
}
//helper methods
public int countWordInString(String input, String word){
if(input == null || input.isEmpty()) {
return 0;
}
int i = 0;
Pattern p = Pattern.compile(word);
Matcher m = p.matcher( input );
while (m.find()) {
i++;
}
return i;
}
protected void createColumns(int noOfColumns){
for(int i = 0; i < noOfColumns; i++){
try {
project.columnModel.addColumn(i, new Column(i, "column" + i), true);
} catch (ModelException e1) {
Assert.fail("Could not create column");
}
}
}
protected void createGrid(int noOfRows, int noOfColumns){
createColumns(noOfColumns);
for(int i = 0; i < noOfRows; i++){
Row row = new Row(noOfColumns);
for(int j = 0; j < noOfColumns; j++){
row.cells.add(new Cell("row" + i + "cell" + j, null));
}
project.rows.add(row);
}
}
protected void createGridWithNullFields(int noOfRows, int noOfColumns, int noOfNullFields){
createColumns(noOfColumns);
if(noOfNullFields > (noOfColumns * noOfRows)) {
noOfNullFields = noOfColumns * noOfRows;
}
int k = 0;
for(int i = 0; i < noOfRows; i++){
Row row = new Row(noOfColumns);
for(int j = 0; j < noOfColumns; j++){
if(k < noOfNullFields) {
row.cells.add(new Cell("", null));
k++;
}else {
row.cells.add(new Cell("row" + i + "cell" + j, null));
}
}
project.rows.add(row);
}
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size) {
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
json.put("tableName", tableName);
List<Column> cols = project.columnModel.columns;
cols.forEach(c -> {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
columnModel.put("name", c.getName());
if(type != null) {
columnModel.put("type", type);
}else {
columnModel.put("type", "VARCHAR");
}
if(size != null) {
columnModel.put("size", size);
}else {
columnModel.put("size", "100");
}
if(type != null) {
columnModel.put("type", type);
}
if(size != null) {
// logger.info(" Size = " + size);
columnModel.put("size", size);
}
columns.put(columnModel);
});
return json;
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue,
boolean allowNull) {
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
json.put("tableName", tableName);
List<Column> cols = project.columnModel.columns;
cols.forEach(c -> {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
columnModel.put("name", c.getName());
if(type != null) {
columnModel.put("type", type);
}else {
columnModel.put("type", "VARCHAR");
}
if(size != null) {
columnModel.put("size", size);
}else {
columnModel.put("size", "100");
}
if(type != null) {
columnModel.put("type", type);
}
if(size != null) {
// logger.info(" Size = " + size);
columnModel.put("size", size);
}
columnModel.put("defaultValue", defaultValue);
columnModel.put("allowNull", allowNull);
columns.put(columnModel);
});
return json;
}
}

4
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -151,6 +151,7 @@ function registerCommands() {
RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand());
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand());
}
function registerOperations() {
@ -468,6 +469,7 @@ function init() {
"scripts/dialogs/templating-exporter-dialog.js",
"scripts/dialogs/column-reordering-dialog.js",
"scripts/dialogs/custom-tabular-exporter-dialog.js",
"scripts/dialogs/sql-exporter-dialog.js",
"scripts/dialogs/expression-column-dialog.js",
"scripts/project/edit-general-metadata-dialog.js",
"scripts/dialogs/http-headers-dialog.js",
@ -507,7 +509,7 @@ function init() {
"styles/dialogs/scatterplot-dialog.less",
"styles/dialogs/column-reordering-dialog.less",
"styles/dialogs/custom-tabular-exporter-dialog.less",
"styles/dialogs/sql-exporter-dialog.less",
"styles/reconciliation/recon-dialog.less",
"styles/reconciliation/standard-service-panel.less",
"styles/reconciliation/extend-data-preview-dialog.less",

15
main/webapp/modules/core/langs/translation-en.json
View File

@ -237,7 +237,7 @@
"help": "Help",
"opt-code-applied": "Option code successfully applied.",
"error-apply-code": "Error applying option code",
"custom-tab-exp": "Custom Tabular Exporter",
"custom-tab-exp": "SQL Exporter",
"content": "Content",
"download": "Download",
"upload": "Upload",
@ -274,6 +274,18 @@
"line-sep": "Line separator",
"upload-to": "Upload to",
"json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options.",
"columnType": "SQL Type",
"for-include-structure-checkbox": "Include Schema",
"for-include-drop-statement-checkbox": "Include Drop Statement",
"for-include-content-checkbox": "Include Content",
"tableNameLabel": "Table Name:",
"sqlExporterTrimColumns": "Trim Column Names",
"sqlExporterIgnoreFacets": "Ignore facets and filters and export all rows",
"sqlExporterOutputEmptyRows":"Output empty row (i.e. all cells null)",
"for-include-if-exist-drop-stmt-checkbox": "Include 'If Exists' in Drop Statement",
"for-null-cell-value-to-empty-str-label": "Convert Null Value to null in Insert"
"choose-export-destination": "Please choose the destination for project export",
"export-to-local": "Export to local",
"export-to-google-drive": "Export to Google Drive"
@ -345,6 +357,7 @@
"excel-xml": "Excel 2007+ (.xlsx)",
"odf": "ODF spreadsheet",
"custom-tabular": "Custom tabular exporter...",
"sql-export": "SQL Exporter...",
"templating": "Templating...",
"warning-align": "You haven't done any schema alignment yet,\n so there is no triple to export.\n\n Use the Freebase > Edit Schema Alignment Skeleton...\n command to align your data with Freebase schemas first.",
"json-invalid": "The JSON you pasted is invalid",

131
main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html Executable file
View File

@ -0,0 +1,131 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div id="sql-exporter-tabs" class="refine-tabs">
<ul>
<li><a href="#sql-exporter-tabs-content" bind="or_dialog_content"></a></li>
<li><a href="#sql-exporter-tabs-download" bind="or_dialog_download"></a></li>
</ul>
<div id="sql-exporter-tabs-content" class="sql-exporter-tabs-content-main">
<div class="grid-layout grid-layout-for-ui layout-normal layout-full"><table>
<!-- <tr>
<td bind="or_dialog_selAndOrd"></td>
<td><span bind="or_dialog_optFor"></span><span bind="columnNameSpan" class="sql-exporter-selected-column"></span></td>
</tr> -->
<tr>
<td width="100%">
<!-- <div bind="columnList" class="sql-exporter-columns"></div> -->
<div class="grid-layout layout-tighter layout-full">
<div class="sql-exporter-columns">
<table bind = "columnListTable">
<thead class="header">
<tr>
<th>Field Name</th>
<th>SQL Type</th>
<th>Size</th>
<th></th>
<th> Allow Null <span style="text-align:center;"><input type="checkbox" bind="allowNullToggleCheckbox" id="allowNullToggleCheckbox" checked /></span></th>
<th> Default</th>
</tr>
</thead>
</table>
</div>
</div>
</td>
</tr>
<tr>
<td colspan="2">
<button class="button" bind="selectAllButton"></button>
<button class="button" bind="deselectAllButton"></button>
</td>
</tr>
<tr>
<td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>
<tr>
<td width="1%"><input type="checkbox" bind="sqlExportOutputEmptyRowsCheckbox" id="sqlExportOutputEmptyRowsCheckboxId" /></td>
<td width="25%"><label for="sqlExportOutputEmptyRowsCheckbox" bind="sqlExportOutputEmptyRowsLabel"></label></td>
<td width="1%"><input type="checkbox" bind="sqlExportAllRowsCheckbox" id="sqlExportAllRowsCheckboxId" /></td>
<td width="25%"><label for="sqlExportAllRowsCheckbox" bind="sqlExportIgnoreFacetsLabel"></label></td>
<td width="1%"><input type="checkbox" bind="sqlExportTrimAllColumnsCheckbox" id="sqlExportTrimAllColumnsCheckboxId" /></td>
<td width="25%"><label for="sqlExportTrimAllColumnsCheckbox" bind="sqlExportTrimAllColumnsLabel"></label></td>
</tr>
</table></div></td>
</tr>
</table></div></div>
<div id="sql-exporter-tabs-download" style="display: none;">
<div>
<table>
<tr>
<td><label for="tableNameTextBox" bind="tableNameLabel"></label></td>
<td width="90%"><input type="text" bind="tableNameTextBox" id="tableNameTextBoxId" size="50%"/></td>
</tr>
<tr>
</tr>
</table>
</div>
<div class="grid-layout grid-layout-for-text layout-tightest"><table>
<tr>
<td><span style="margin:2px; padding:2px;"></span></td>
</tr>
<tr>
<td width="1%"><input type="checkbox" bind="includeStructureCheckbox" id="includeStructureCheckboxId" checked/></td>
<td><label for="includeStructureCheckboxId" bind="includeStructureLabel"></label></td>
</tr>
<tr>
<td width="1%"><input type="checkbox" bind="includeDropStatementCheckbox" id="includeDropStatementCheckboxId"/></td>
<td><label for="includeDropStatementCheckboxId" bind="includeDropStatementLabel"></label></td>
</tr>
<tr>
<td width="1%"><input type="checkbox" bind="includeIfExistDropStatementCheckbox" id="includeIfExistDropStatementCheckboxId"/></td>
<td><label for="includeIfExistDropStatementCheckboxId" bind="includeIfExistDropStatementLabel"></label></td>
</tr>
<tr>
<td><span style="margin:2px; padding:2px;"></span></td>
</tr>
<tr>
<td width="1%"><input type="checkbox" bind="includeContentCheckbox" id="includeContentCheckboxId" checked/></td>
<td><label for="includeContentCheckboxId" bind="includeContentLabel"></label></td>
</tr>
<tr>
<td width="1%"><input type="checkbox" bind="nullCellValueToEmptyStringCheckbox" id="nullCellValueToEmptyStringCheckboxId" checked/></td>
<td><label for="nullCellValueToEmptyStringCheckbox" bind="nullCellValueToEmptyStringLabel"></label></td>
</tr>
<tr>
<td colspan="2"><div class="grid-layout grid-layout-for-text layout-tightest layout-full"><table><tr>
<td width="100%"> </td>
<td width="1%"><button class="button" bind="downloadPreviewButton"></button></td>
<td width="1%"><button class="button button-primary" bind="downloadButton"></button></td>
</tr></table></div></td>
</tr>
</table></div></div>
<div id="sql-exporter-tabs-code" style="display: none;">
<textarea class="sql-exporter-code" bind="optionCodeInput"></textarea>
</div>
</div>
</div>
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-tightest layout-full"><table><tr>
<td><button class="button" bind="cancelButton"></button></td>
</tr></table></div></div>
</div>

495
main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js Executable file
View File

@ -0,0 +1,495 @@
/*
* Copyright (c) 2017, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
function SqlExporterDialog(options) {
options = options || {
format: 'sql',
encoding: 'UTF-8',
outputBlankRows: false,
columns: null
};
this._columnOptionMap = {};
this._createDialog(options);
}
SqlExporterDialog.uploadTargets = [];
SqlExporterDialog.prototype._createDialog = function(options) {
var self = this;
this._dialog = $(DOM.loadHTML("core", "scripts/dialogs/sql-exporter-dialog.html"));
this._elmts = DOM.bind(this._dialog);
this._level = DialogSystem.showDialog(this._dialog);
this._elmts.dialogHeader.html($.i18n._('core-dialogs')["custom-tab-exp"]);
this._elmts.or_dialog_content.html($.i18n._('core-dialogs')["content"]);
this._elmts.or_dialog_download.html($.i18n._('core-dialogs')["download"]);
this._elmts.selectAllButton.html($.i18n._('core-buttons')["select-all"]);
this._elmts.deselectAllButton.html($.i18n._('core-buttons')["deselect-all"]);
this._elmts.downloadPreviewButton.html($.i18n._('core-buttons')["preview"]);
this._elmts.downloadButton.html($.i18n._('core-buttons')["download"]);
this._elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]);
// this._elmts.nextButton.html($.i18n._('core-buttons')["next"]);
this._elmts.tableNameLabel.html($.i18n._('core-dialogs')["tableNameLabel"]);
this._elmts.includeStructureLabel.html($.i18n._('core-dialogs')["for-include-structure-checkbox"]);
this._elmts.includeDropStatementLabel.html($.i18n._('core-dialogs')["for-include-drop-statement-checkbox"]);
this._elmts.includeContentLabel.html($.i18n._('core-dialogs')["for-include-content-checkbox"]);
this._elmts.includeIfExistDropStatementLabel.html($.i18n._('core-dialogs')["for-include-if-exist-drop-stmt-checkbox"]);
this._elmts.nullCellValueToEmptyStringLabel.html($.i18n._('core-dialogs')["for-null-cell-value-to-empty-str-label"]);
this._elmts.sqlExportIgnoreFacetsLabel.html($.i18n._('core-dialogs')["sqlExporterIgnoreFacets"]);
this._elmts.sqlExportTrimAllColumnsLabel.html($.i18n._('core-dialogs')["sqlExporterTrimColumns"]);
this._elmts.sqlExportOutputEmptyRowsLabel.html($.i18n._('core-dialogs')["sqlExporterOutputEmptyRows"]);
$("#sql-exporter-tabs-content").css("display", "");
$("#sql-exporter-tabs-download").css("display", "");
$("#sql-exporter-tabs").tabs();
for (var i = 0; i < theProject.columnModel.columns.length; i++) {
var column = theProject.columnModel.columns[i];
var name = column.name;
var rowId = "sql-exporter-dialog-row" + i;
var selectBoxName = 'selectBoxRow' + i;
var sizeInputName = 'sizeInputRow' + i;
var applyAllBtnName = 'applyAllBtn' + i;
var allowNullChkBoxName = 'allowNullChkBox' + i;
var defaultValueTextBoxName = 'defaultValueTextBox' + i;
var sel = $('<select>').appendTo('body');
sel.append($("<option>").attr('value','VARCHAR').text('VARCHAR'));
sel.append($("<option>").attr('value','TEXT').text('TEXT'));
sel.append($("<option>").attr('value','INT').text('INT'));
sel.append($("<option>").attr('value','NUMERIC').text('NUMERIC'));
sel.append($("<option>").attr('value','CHAR').text('CHAR'));
sel.append($("<option>").attr('value','DATE').text('DATE'));
sel.append($("<option>").attr('value','TIMESTAMP').text('TIMESTAMP'));
sel.attr('id', selectBoxName);
sel.attr('rowIndex', i);
sel.addClass('typeSelectClass');
sel.css({'width':'95%'});
$(sel).on('change', function() {
var rowIndex = this.getAttribute('rowIndex');
if (this.value === 'VARCHAR' || this.value === 'CHAR' || this.value === 'NUMERIC' || this.value === 'INT') {
$('#sizeInputRow'+ rowIndex).prop("disabled", false);
}else{
$('#sizeInputRow'+ rowIndex).val("");
$('#sizeInputRow'+ rowIndex).prop("disabled", true);
}
});
//create and add Row
var row = $('<tr>')
.addClass("sql-exporter-dialog-row")
.attr('id', rowId)
.attr("column", name)
.attr("rowIndex", i)
.appendTo(this._elmts.columnListTable);
//create and add column
var columnCell = $('<td>')
.attr('width', '150px')
.appendTo(row);
$('<input>')
.attr('type', 'checkbox')
.attr('checked', 'checked')
.addClass("columnNameCheckboxStyle")
.appendTo(columnCell);
$('<span>')
.text(name)
.appendTo(columnCell);
var typeCell = $('<td>')
.attr('width', '150px')
.appendTo(row);
sel.appendTo(typeCell);
var sizeCell = $('<td>')
.attr('width', '20px')
.appendTo(row);
$('<input>')
.attr('type', 'text')
.attr('size', '8px')
.attr('id', sizeInputName)
.addClass("sql-exporter-dialog-input")
.appendTo(sizeCell);
var applyAllCell = $('<td>')
.attr('width', '60px')
.appendTo(row);
$('<input>')
.attr('type', 'button')
.attr('value', 'Apply All')
.attr('id', applyAllBtnName)
.attr("rowIndex", i)
.appendTo(applyAllCell);
var nullableCell = $('<td>')
.attr('width', '90px')
.addClass("allowNullCellStyle")
.appendTo(row);
$('<input>')
.attr('type', 'checkbox')
.attr('checked', 'checked')
.attr('id', allowNullChkBoxName)
.attr("rowIndex", i)
.addClass("allowNullCheckboxStyle")
.appendTo(nullableCell);
var defValueCell = $('<td>')
.attr('width', '30px')
.appendTo(row);
$('<input>')
.attr('type', 'text')
.attr('size', '8px')
.attr('id', defaultValueTextBoxName)
.attr("rowIndex", i)
.addClass("defaultValueTextBoxStyle")
.appendTo(defValueCell);
$('#' + applyAllBtnName).on('click', function() {
var rowIndex = this.getAttribute('rowIndex');
var typeValue = $("#selectBoxRow" + rowIndex).val();
var sizeValue = $("#sizeInputRow" + rowIndex).val();
$('select.typeSelectClass').each(function() {
//alert("Value:" + this.value + " RowIndex:" + rowIndex + " TypeValue:" + typeValue + "" + this.value);
var rowId = this.getAttribute('rowIndex');
var id = this.getAttribute('id');
if(rowIndex !== rowId){
$("#" + id).val(typeValue);
}
});
$('input.sql-exporter-dialog-input').each(function() {
var rowId = this.getAttribute('rowIndex');
var id = this.getAttribute('id');
if(rowIndex !== rowId){
$("#" + id).val(sizeValue);
}
});
});
$('#' + allowNullChkBoxName).on('click', function() {
var rowIndex = this.getAttribute('rowIndex');
var id = this.getAttribute('id');
var checked = $(this).is(':checked');;
if(checked == false){
$('#defaultValueTextBox'+ rowIndex).prop("disabled", false);
}else{
$('#defaultValueTextBox'+ rowIndex).val("");
$('#defaultValueTextBox'+ rowIndex).prop("disabled", true);
}
});
this._columnOptionMap[name] = {
name: name,
type: '',
size: ''
};
}
this._elmts.allowNullToggleCheckbox.click(function() {
var checked = $(this).is(':checked');
if(checked == true){
$("input:checkbox[class=allowNullCheckboxStyle]").each(function () {
$(this).attr('checked', true);
});
$("input:text[class=defaultValueTextBoxStyle]").each(function () {
$(this).attr('disabled', true);
});
}else{
$("input:checkbox[class=allowNullCheckboxStyle]").each(function () {
$(this).attr('checked', false);
});
$("input:text[class=defaultValueTextBoxStyle]").each(function () {
$(this).attr('disabled', false);
});
}
});
this._elmts.selectAllButton.click(function() {
$("input:checkbox[class=columnNameCheckboxStyle]").each(function () {
$(this).attr('checked', true);
});
self._updateOptionCode();
});
this._elmts.deselectAllButton.click(function() {
$("input:checkbox[class=columnNameCheckboxStyle]").each(function () {
$(this).attr('checked', false);
});
self._updateOptionCode();
});
this._elmts.includeStructureCheckbox.click(function() {
var checked = $(this).is(':checked');
//alert('checked ' + checked);
if(checked == true){
$('#includeDropStatementCheckboxId').removeAttr("disabled");
$('#includeIfExistDropStatementCheckboxId').removeAttr("disabled");
}else{
$('#includeDropStatementCheckboxId').attr("disabled", true);
$('#includeIfExistDropStatementCheckboxId').attr("disabled", true);
}
});
this._elmts.includeContentCheckbox.click(function() {
var checked = $(this).is(':checked');
if(checked == true){
$('#nullCellValueToEmptyStringCheckboxId').removeAttr("disabled");
}else{
$('#nullCellValueToEmptyStringCheckboxId').attr("disabled", true);
}
});
this._elmts.cancelButton.click(function() { self._dismiss(); });
this._elmts.downloadButton.click(function() { self._download(); });
this._elmts.downloadPreviewButton.click(function(evt) { self._previewDownload(); });
this._configureUIFromOptionCode(options);
this._updateOptionCode();
};
SqlExporterDialog.prototype._configureUIFromOptionCode = function(options) {
this._elmts.tableNameTextBox.val(theProject.metadata.name);
this._elmts.sqlExportOutputEmptyRowsCheckbox.attr('checked', 'checked');
this._elmts.sqlExportTrimAllColumnsCheckbox.attr('checked', 'checked');
this._elmts.nullCellValueToEmptyStringLabel.attr('checked', 'checked');
$("input:text[class=defaultValueTextBoxStyle]").each(function () {
$(this).prop("disabled", true);
});
};
SqlExporterDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
SqlExporterDialog.prototype._previewDownload = function() {
this._postExport(true);
};
SqlExporterDialog.prototype._download = function() {
var result = this._postExport(false);
// alert("result::" + result);
if(result == true){
this._dismiss();
}
};
SqlExporterDialog.prototype._postExport = function(preview) {
var self = this;
var exportAllRowsCheckbox = this._elmts.sqlExportAllRowsCheckbox[0].checked;
var options = this._getOptionCode();
if(options.columns == null || options.columns.length == 0){
alert("Please select at least one column...");
return false;
}
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
var format = options.format;
var encoding = options.encoding;
delete options.format;
delete options.encoding;
if (preview) {
options.limit = 10;
}
// var ext = SqlExporterDialog.formats[format].extension;
var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql");
$('<input />')
.attr("name", "options")
.attr("value", JSON.stringify(options))
.appendTo(form);
if (encoding) {
$('<input />')
.attr("name", "encoding")
.attr("value", encoding)
.appendTo(form);
}
if (!preview) {
$('<input />')
.attr("name", "contentType")
.attr("value", "application/x-unknown") // force download
.appendTo(form);
}
// alert("form::" + form);
document.body.appendChild(form);
window.open("about:blank", "refine-export");
form.submit();
document.body.removeChild(form);
return true;
};
SqlExporterDialog.prototype._prepareSqlExportRowsForm = function(format, includeEngine, ext) {
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
var form = document.createElement("form");
$(form)
.css("display", "none")
.attr("method", "post")
.attr("action", "command/core/export-rows/" + name + ((ext) ? ("." + ext) : ""))
.attr("target", "refine-export");
$('<input />')
.attr("name", "project")
.attr("value", theProject.id)
.appendTo(form);
$('<input />')
.attr("name", "format")
.attr("value", format)
.appendTo(form);
if (includeEngine) {
$('<input />')
.attr("name", "engine")
.attr("value", JSON.stringify(ui.browsingEngine.getJSON()))
.appendTo(form);
}
return form;
};
SqlExporterDialog.prototype._selectColumn = function(columnName) {
this._elmts.columnNameSpan.text(columnName);
var columnOptions = this._columnOptionMap[columnName];
alert("in _selectColumn:column type::" + columnOptions.type);
};
SqlExporterDialog.prototype._updateCurrentColumnOptions = function() {
// var selectedColumnName = this._elmts.columnList.find('.sql-exporter-dialog-column.selected').attr('column');
// //alert("_updateCurrentColumnOptions::" + selectedColumnName);
// var columnOptions = this._columnOptionMap[selectedColumnName];
// columnOptions.type= this._elmts.columnOptionPane.find('input[name="sql-exporter-type"]:checked').val();
};
SqlExporterDialog.prototype._updateOptionCode = function() {
this._elmts.optionCodeInput.val(JSON.stringify(this._getOptionCode(), null, 2));
};
SqlExporterDialog.prototype._getOptionCode = function() {
var options = {
//format: this._dialog.find('input[name="sql-exporter-download-format"]:checked').val()
};
var unescapeJavascriptString = function(s) {
try {
return JSON.parse('"' + s + '"');
} catch (e) {
// We're not handling the case where the user doesn't escape double quotation marks.
return s;
}
};
options.format = 'sql';
options.separator = ';';
options.encoding = 'UTF-8';
options.outputBlankRows = this._elmts.sqlExportOutputEmptyRowsCheckbox[0].checked;
options.includeStructure = this._elmts.includeStructureCheckbox[0].checked;
options.includeDropStatement = this._elmts.includeDropStatementCheckbox[0].checked;
options.includeContent = this._elmts.includeContentCheckbox[0].checked;
options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '_');
options.trimColumnNames = this._elmts.sqlExportTrimAllColumnsCheckbox[0].checked;
options.convertNulltoEmptyString = this._elmts.nullCellValueToEmptyStringCheckbox[0].checked;
options.includeIfExistWithDropStatement = this._elmts.includeIfExistDropStatementCheckbox[0].checked;
options.columns = [];
var self = this;
this._elmts.columnListTable.find('.sql-exporter-dialog-row').each(function() {
if ($(this).find('input[type="checkbox"]')[0].checked) {
var name = this.getAttribute('column');
var rowIndex = this.getAttribute('rowIndex');
var selectedValue = $('#selectBoxRow' + rowIndex).val();
var typeSize = 0;
if(selectedValue === 'VARCHAR' || selectedValue === 'CHAR' || selectedValue === 'INT' || selectedValue === 'NUMERIC'){
typeSize = $('#sizeInputRow' + rowIndex).val();
// alert("typeSize::" + typeSize);
}
var allowNullChkBoxName = $('#allowNullChkBox' + rowIndex).is(':checked');
var defaultValueTextBoxName = $('#defaultValueTextBox' + rowIndex).val();
// alert("allowNullChkBoxName::" + allowNullChkBoxName);
// alert("defaultValueTextBoxName::" + defaultValueTextBoxName);
var fullColumnOptions = self._columnOptionMap[name];
var columnOptions = {
name: name,
type: selectedValue,
size: typeSize,
allowNull: allowNullChkBoxName,
defaultValue: defaultValueTextBoxName,
nullValueToEmptyStr: options.convertNulltoEmptyString
};
// alert('checked type ' + columnIndex + ' =' + check_value);
options.columns.push(columnOptions);
}
});
//alert('options:' + options);
return options;
};

6
main/webapp/modules/core/scripts/project/exporters.js
View File

@ -86,6 +86,11 @@ ExporterManager.MenuItems = [
"label": $.i18n._('core-project')["custom-tabular"],
"click": function() { new CustomTabularExporterDialog(); }
},
{
"id" : "core/export-sql",
"label": $.i18n._('core-project')["sql-export"],
"click": function() { new SqlExporterDialog(); }
},
{
"id" : "core/export-templating",
"label": $.i18n._('core-project')["templating"],
@ -131,6 +136,7 @@ ExporterManager.handlers.exportRows = function(format, ext) {
ExporterManager.prepareExportRowsForm = function(format, includeEngine, ext) {
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
//alert("name:" + name);
var form = document.createElement("form");
$(form)
.css("display", "none")

101
main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less Normal file
View File

@ -0,0 +1,101 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
#sql-exporter-tabs .ui-tabs-panel {
padding: @padding_looser;
}
.sql-exporter-tabs-content-main {
}
.sql-exporter-columns, .sql-exporter-column-options {
/*border: 1px solid @chrome_primary;*/
height: 8em;
padding: @padding_loose;
}
.sql-exporter-columns {
min-height: 200px;
overflow: auto;
}
.sql-exporter-selected-column {
font-weight: bold;
border: 1px solid @chrome_primary;
background: @chrome_primary;
padding: @padding_tighter;
.rounded_corners();
}
.sql-exporter-dialog-column {
/* border: 1px solid @chrome_primary;*/
background: @fill_secondary;
padding: @padding_tighter;
margin: @padding_tight;
cursor: move;
.rounded_corners();
}
.sql-exporter-dialog-column.selected {
background: @chrome_primary;
font-weight: bold;
}
textarea.sql-exporter-code {
width: 100%;
border: 1px solid @chrome_primary;
height: 25em;
}
.sql-exporter-dialog-row {
margin-top: 2px;
padding-top: 2px;
margin-bottom: 2px;
margin-top: 2px;
}
tr.sql-exporter-dialog-row td {
border:1px dashed lightblue;
margin: 2px;
}
.sql-exporter-dialog-input {
margin:2px;
border: solid 2px lightblue;
}
.defaultValueTextBoxStyle {
margin:2px;
}
.allowNullCellStyle {
text-align: center;
vertical-align: middle;
}
.typeSelectClass {
margin-left:2px;
}