From 3834c27c8fab680ab532359db50e2d269133afe9 Mon Sep 17 00:00:00 2001 From: TonyO Date: Sat, 24 Mar 2018 22:33:41 -0500 Subject: [PATCH 1/6] SQL Export Code Commit --- .../refine/exporters/ExporterRegistry.java | 4 + .../exporters/sql/SqlCreateBuilder.java | 130 ++++++ .../google/refine/exporters/sql/SqlData.java | 61 +++ .../refine/exporters/sql/SqlExporter.java | 159 +++++++ .../exporters/sql/SqlInsertBuilder.java | 151 +++++++ .../tests/exporters/sql/SqlExporterTests.java | 304 +++++++++++++ .../webapp/modules/core/MOD-INF/controller.js | 3 +- .../modules/core/langs/translation-en.json | 11 +- .../scripts/dialogs/sql-exporter-dialog.html | 110 +++++ .../scripts/dialogs/sql-exporter-dialog.js | 408 ++++++++++++++++++ .../modules/core/scripts/project/exporters.js | 6 + .../styles/dialogs/sql-exporter-dialog.less | 94 ++++ 12 files changed, 1438 insertions(+), 3 deletions(-) create mode 100755 main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java create mode 100755 main/src/com/google/refine/exporters/sql/SqlData.java create mode 100755 main/src/com/google/refine/exporters/sql/SqlExporter.java create mode 100755 main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java create mode 100644 main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java create mode 100755 main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html create mode 100755 main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js create mode 100644 main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less diff --git a/main/src/com/google/refine/exporters/ExporterRegistry.java b/main/src/com/google/refine/exporters/ExporterRegistry.java index a8f321157..69bc92103 100644 --- a/main/src/com/google/refine/exporters/ExporterRegistry.java +++ b/main/src/com/google/refine/exporters/ExporterRegistry.java @@ -36,6 +36,8 @@ package com.google.refine.exporters; import java.util.HashMap; import java.util.Map; +import com.google.refine.exporters.sql.SqlExporter; + abstract public class ExporterRegistry { static final private Map s_formatToExporter = new HashMap(); @@ -52,6 +54,8 @@ abstract public class ExporterRegistry { s_formatToExporter.put("html", new HtmlTableExporter()); s_formatToExporter.put("template", new TemplatingExporter()); + + s_formatToExporter.put("sql", new SqlExporter()); } static public void registerExporter(String format, Exporter exporter) { diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java new file mode 100755 index 000000000..588b208ba --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.util.JSONUtilities; + +public class SqlCreateBuilder { + + private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder"); + + private String table; + + @SuppressWarnings("unused") + private List columns; + + private JSONObject options; + + public SqlCreateBuilder(String table, List columns, JSONObject options) { + this.table = table; + this.columns = columns; + this.options = options; + } + + public String getCreateSQL() { + StringBuffer createSB = new StringBuffer(); + + JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); + + int count = columnOptionArray.length(); + + for (int i = 0; i < count; i++) { + JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i); + if (columnOptions != null) { + String name = JSONUtilities.getString(columnOptions, "name", null); + String type = JSONUtilities.getString(columnOptions, "type", "VARCHAR"); + String size = JSONUtilities.getString(columnOptions, "size", ""); + if (name != null) { + createSB.append(name + " "); + + if (type.equals("VARCHAR")) { + if (size.isEmpty()) { + size = "255"; + } + createSB.append(type + "(" + size + ")"); + + } else if (type.equals("CHAR")) { + if (size.isEmpty()) { + size = "10"; + } + createSB.append(type + "(" + size + ")"); + + } else if (type.equals("INT") || type.equals("INTEGER")) { + if (size.isEmpty()) { + createSB.append(type); + } else { + createSB.append(type + "(" + size + ")"); + } + + } else if (type.equals("NUMERIC")) { + if (size.isEmpty()) { + createSB.append(type); + } else { + createSB.append(type + "(" + size + ")"); + } + } else { + createSB.append(type); + } + + if (i < count - 1) { + createSB.append(","); + } + createSB.append("\n"); + } + } + } + + StringBuffer sql = new StringBuffer(); + + boolean includeDrop = JSONUtilities.getBoolean(options, "includeDropStatement", false); + if (includeDrop) { + sql.append("DROP TABLE " + table + ";\n"); + } + + sql.append("CREATE TABLE ").append(table); + sql.append(" (").append("\n"); + sql.append(createSB.toString()); + sql.append(")").append(";" + "\n"); + + String createSQL = sql.toString(); + if(logger.isDebugEnabled()){ + logger.debug("Create SQL Generated Successfully...{}", createSQL); + } + return createSQL; + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlData.java b/main/src/com/google/refine/exporters/sql/SqlData.java new file mode 100755 index 000000000..6e595b733 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlData.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + + +public class SqlData { + + final public String columnName; + final public Object value; + final public String text; + + public SqlData(String columnName, Object value, String text) { + this.columnName = columnName; + this.value = value; + this.text = text; + + } + + + public String getColumnName() { + return columnName; + } + + + public Object getValue() { + return value; + } + + + public String getText() { + return text; + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java new file mode 100755 index 000000000..67fa176aa --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.exporters.CustomizableTabularExporterUtilities; +import com.google.refine.exporters.TabularSerializer; +import com.google.refine.exporters.WriterExporter; +import com.google.refine.model.Project; +import com.google.refine.util.JSONUtilities; + +public class SqlExporter implements WriterExporter { + + public static final String NO_COL_SELECTED_ERROR = "****NO COLUMNS SELECTED****"; + public static final String NO_OPTIONS_PRESENT_ERROR = "****NO OPTIONS PRESENT****"; + + private final static Logger logger = LoggerFactory.getLogger("SqlExporter"); + + private List columnNames = new ArrayList(); + private List> sqlDataList = new ArrayList>(); + private JSONObject sqlOptions; + + @Override + public String getContentType() { + return "text/plain"; + } + + @Override + public void export(final Project project, Properties params, Engine engine, final Writer writer) + throws IOException { + + TabularSerializer serializer = new TabularSerializer() { + + @Override + public void startFile(JSONObject options) { + sqlOptions = options; + //logger.info("setting options::{}", sqlOptions); + } + + @Override + public void endFile() { + try { + if (columnNames.isEmpty()) { + writer.write(NO_COL_SELECTED_ERROR); + logger.error("No Columns Selected!!"); + return; + } + if (sqlOptions == null) { + writer.write(NO_OPTIONS_PRESENT_ERROR); + logger.error("No Options Selected!!"); + return; + } + String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName(); + + Object tableNameManual = sqlOptions.get("tableName"); + + if (tableNameManual != null && !tableNameManual.toString().isEmpty()) { + tableName = tableNameManual.toString(); + } + + SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions); + SqlInsertBuilder insertBuilder = new SqlInsertBuilder(tableName, columnNames, sqlDataList, + sqlOptions); + + final boolean includeStructure = sqlOptions == null ? true + : JSONUtilities.getBoolean(sqlOptions, "includeStructure", true); + + final boolean includeContent = sqlOptions == null ? true + : JSONUtilities.getBoolean(sqlOptions, "includeContent", true); + + if (includeStructure) { + String sqlCreateStr = createBuilder.getCreateSQL(); + writer.write(sqlCreateStr); + + } + + if (includeContent) { + String sqlInsertStr = insertBuilder.getInsertSQL(); + writer.write(sqlInsertStr); + } + + if (logger.isDebugEnabled()) { + logger.debug("sqlOptions::{}", sqlOptions); + } + + columnNames = new ArrayList(); + sqlDataList = new ArrayList>(); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void addRow(List cells, boolean isHeader) { + + if (isHeader) { + for (CellData cellData : cells) { + columnNames.add(cellData.text); + } + + } else { + ArrayList values = new ArrayList<>(); + for (CellData cellData : cells) { + + if (cellData != null && cellData.text != null) { + SqlData newSql = new SqlData(cellData.columnName, cellData.value, cellData.text); + values.add(newSql); + + } + + } + sqlDataList.add(values); + } + + } + }; + + CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer); + } +} diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java new file mode 100755 index 000000000..7e9e77a1b --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.util.JSONUtilities; + +public class SqlInsertBuilder { + + private static final Logger logger = LoggerFactory.getLogger("SQLInsertBuilder"); + + private String table; + + private List columns; + + private List> sqlDataList; + + private JSONObject options; + + /** + * + * @param table + * @param columns + * @param rows + * @param options + */ + public SqlInsertBuilder(String table, List columns, List> rows, JSONObject options) { + this.table = table; + this.columns = columns; + this.sqlDataList = rows; + this.options = options; + } + + /** + * Get Insert Sql + * @return + */ + public String getInsertSQL() { + // logger.info("options values::::{}", options); + JSONArray columnOptionArray = options == null ? null : + JSONUtilities.getArray(options, "columns"); + //logger.info("columnOptionArray::::{}", columnOptionArray); + + Map colOptionsMap = new HashMap(); + if(columnOptionArray != null) { + columnOptionArray.forEach(c -> { + JSONObject json = (JSONObject)c; + colOptionsMap.put("" + json.get("name"), json); + }); + } + + String colNamesWithSep = columns.stream() + .collect(Collectors.joining(",")); + StringBuffer values = new StringBuffer(); + + int idx = 0; + for(ArrayList sqlCellData : sqlDataList) { + StringBuilder rowValue = new StringBuilder(); + //logger.info(" row.size:{}", row.size()); + for(SqlData val : sqlCellData) { + + JSONObject jsonOb = colOptionsMap.get(val.getColumnName()); + String type = (String)jsonOb.get("type"); + if(type == null) { + type = "VARCHAR"; + } + if(type.equals("VARCHAR") || type.equals("CHAR") || type.equals("TEXT")) { + + String value = "'" + val.text + "'"; + rowValue.append(value); + }else { + rowValue.append(val.text); + } + + rowValue.append(","); + //logger.info("jsonObject:{}", jsonOb); + + } + idx++; + String rowValString = rowValue.toString(); + rowValString = rowValString.substring(0, rowValString.length() - 1); + + values.append("( "); + values.append(rowValString); + values.append(" )"); + if(idx < sqlDataList.size()) { + values.append(","); + } + values.append("\n"); +// logger.info("running values:{}", values.toString()); + + } + + + String valuesString = values.toString(); + valuesString = valuesString.substring(0, valuesString.length() - 1); + + + StringBuffer sql = new StringBuffer(); + + sql.append("INSERT INTO ").append(table); + sql.append(" ("); + sql.append(colNamesWithSep); + sql.append(") VALUES ").append("\n"); + sql.append(valuesString); + + String sqlString = sql.toString(); + if(logger.isDebugEnabled()) { + logger.debug("Insert Statement Generated Successfully...{}", sqlString); + } + return sqlString; + } + +} diff --git a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java new file mode 100644 index 000000000..e78ebf069 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java @@ -0,0 +1,304 @@ +/* + +Copyright 2018, Tony Opara. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.tests.exporters.sql; + +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.exporters.sql.SqlCreateBuilder; +import com.google.refine.exporters.sql.SqlExporter; +import com.google.refine.exporters.sql.SqlInsertBuilder; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.model.Row; +import com.google.refine.model.medadata.ProjectMetadata; +import com.google.refine.tests.ProjectManagerStub; +import com.google.refine.tests.RefineTest; + + +public class SqlExporterTests extends RefineTest { + + private static final String TEST_PROJECT_NAME = "SQL_EXPORTER_TEST_PROJECT"; + + @Override + @BeforeTest + public void init() { + logger = LoggerFactory.getLogger(this.getClass()); + } + + //dependencies + StringWriter writer; + ProjectMetadata projectMetadata; + Project project; + Engine engine; + Properties options; + SqlCreateBuilder sqlCreateBuilder; + SqlInsertBuilder sqlInsertBuilder; + + //System Under Test + SqlExporter SUT; + + @BeforeMethod + public void SetUp(){ + SUT = new SqlExporter(); + writer = new StringWriter(); + ProjectManager.singleton = new ProjectManagerStub(); + projectMetadata = new ProjectMetadata(); + project = new Project(); + projectMetadata.setName(TEST_PROJECT_NAME); + ProjectManager.singleton.registerProject(project, projectMetadata); + engine = new Engine(project); + options = mock(Properties.class); + } + + @AfterMethod + public void TearDown(){ + SUT = null; + writer = null; + ProjectManager.singleton.deleteProject(project.id); + project = null; + projectMetadata = null; + engine = null; + options = null; + sqlCreateBuilder = null; + sqlInsertBuilder = null; + } + + @Test + public void testExportSimpleSql(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + String optionsString = createOptionsFromProject(tableName, null,null).toString(); + when(options.getProperty("options")).thenReturn(optionsString); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + // logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + //logger.info("checkResult1 =" + checkResult); + checkResult = result.contains("INSERT INTO " + tableName); + // logger.info("checkResult2 =" + checkResult); + Assert.assertEquals(checkResult, true); + + } + + @Test + public void testExportSqlNoSchema(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null,null); + optionsJson.put("includeStructure", false); + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + // logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + //logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, false); + + checkResult = result.contains("INSERT INTO " + tableName); + Assert.assertEquals(checkResult, true); + + } + + @Test + public void testExportSqlNoContent(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeContent", false); + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + // logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, true); + + checkResult = result.contains("INSERT INTO " + tableName); + Assert.assertEquals(checkResult, false); + + } + + @Test + public void testExportSqlIncludeSchemaWithDropStmt(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR); + + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, true); + + checkResult = result.contains("INSERT INTO " + tableName ); + Assert.assertEquals(checkResult, true); + + checkResult = result.contains("DROP TABLE " + tableName + ";"); + Assert.assertEquals(checkResult, true); + + } + + @Test + public void testGetCreateSql(){ + createGrid(3,3); + String tableName = "sql_table_test"; + String type = "CHAR"; + String size = "2"; + JSONObject optionsJson = createOptionsFromProject(tableName, type, size); + // logger.info("Options:: = " + optionsJson.toString()); + List columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList()); + + sqlCreateBuilder = new SqlCreateBuilder(tableName, columns, optionsJson); + String createSql = sqlCreateBuilder.getCreateSQL(); + //logger.info("createSql = \n" + createSql); + Assert.assertNotNull(createSql); + boolean result = createSql.contains(type + "(" + size + ")"); + Assert.assertEquals(result, true); + + } + + //helper methods + protected void createColumns(int noOfColumns){ + for(int i = 0; i < noOfColumns; i++){ + try { + project.columnModel.addColumn(i, new Column(i, "column" + i), true); + } catch (ModelException e1) { + Assert.fail("Could not create column"); + } + } + } + + protected void createGrid(int noOfRows, int noOfColumns){ + createColumns(noOfColumns); + + for(int i = 0; i < noOfRows; i++){ + Row row = new Row(noOfColumns); + for(int j = 0; j < noOfColumns; j++){ + row.cells.add(new Cell("row" + i + "cell" + j, null)); + } + project.rows.add(row); + } + } + + protected JSONObject createOptionsFromProject(String tableName, String type, String size) { + + JSONObject json = new JSONObject(); + JSONArray columns = new JSONArray(); + json.put("columns", columns); + json.put("tableName", tableName); + + List cols = project.columnModel.columns; + + cols.forEach(c -> { + //logger.info("Column Name = " + c.getName()); + JSONObject columnModel = new JSONObject(); + columnModel.put("name", c.getName()); + columnModel.put("type", "VARCHAR"); + columnModel.put("size", "100"); + if(type != null) { + columnModel.put("type", type); + } + if(size != null) { + // logger.info(" Size = " + size); + columnModel.put("size", size); + } + + columns.put(columnModel); + + }); + + return json; + } + + +} diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index 23e5304eb..a11a5f238 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -467,6 +467,7 @@ function init() { "scripts/dialogs/templating-exporter-dialog.js", "scripts/dialogs/column-reordering-dialog.js", "scripts/dialogs/custom-tabular-exporter-dialog.js", + "scripts/dialogs/sql-exporter-dialog.js", "scripts/dialogs/expression-column-dialog.js", "scripts/project/edit-general-metadata-dialog.js", "scripts/dialogs/http-headers-dialog.js", @@ -506,7 +507,7 @@ function init() { "styles/dialogs/scatterplot-dialog.less", "styles/dialogs/column-reordering-dialog.less", "styles/dialogs/custom-tabular-exporter-dialog.less", - + "styles/dialogs/sql-exporter-dialog.less", "styles/reconciliation/recon-dialog.less", "styles/reconciliation/standard-service-panel.less", "styles/reconciliation/extend-data-preview-dialog.less", diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index aff50f5bb..be9ca529a 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -237,7 +237,7 @@ "help": "Help", "opt-code-applied": "Option code successfully applied.", "error-apply-code": "Error applying option code", - "custom-tab-exp": "Custom Tabular Exporter", + "custom-tab-exp": "SQL Exporter", "content": "Content", "download": "Download", "upload": "Upload", @@ -273,7 +273,13 @@ "char-enc": "Character encoding", "line-sep": "Line separator", "upload-to": "Upload to", - "json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options." + "json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options.", + "columnType": "SQL Type", + "for-include-structure-checkbox": "Include Schema", + "for-include-drop-statement-checkbox": "Include Drop Statement", + "for-include-content-checkbox": "Include Content", + "tableNameLabel": "Table Name:" + }, "core-facets": { "remove-facet": "Remove this facet", @@ -344,6 +350,7 @@ "triple-loader": "Triple loader", "mqlwrite": "MQLWrite", "custom-tabular": "Custom tabular exporter...", + "sql-export": "SQL Exporter...", "templating": "Templating...", "warning-align": "You haven't done any schema alignment yet,\n so there is no triple to export.\n\n Use the Freebase > Edit Schema Alignment Skeleton...\n command to align your data with Freebase schemas first.", "json-invalid": "The JSON you pasted is invalid", diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html new file mode 100755 index 000000000..136ef7a53 --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html @@ -0,0 +1,110 @@ +
+
+
+
+
    +
    • +
    • +
+ +
+
+ + + + + + + + + + + +
+ + +
+
+ + + + + + + + + + +
Field NameSQL TypeSize
+
+ +
+
+ + +
+ + + + + + +
+ + + + + + +
+
+
+ +
+
\ No newline at end of file diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js new file mode 100755 index 000000000..411003f9f --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2017, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +function SqlExporterDialog(options) { + options = options || { + format: 'sql', + encoding: 'UTF-8', + outputBlankRows: false, + columns: null + }; + + this._columnOptionMap = {}; + this._createDialog(options); + } + + + SqlExporterDialog.uploadTargets = []; + + SqlExporterDialog.prototype._createDialog = function(options) { + var self = this; + + this._dialog = $(DOM.loadHTML("core", "scripts/dialogs/sql-exporter-dialog.html")); + this._elmts = DOM.bind(this._dialog); + this._level = DialogSystem.showDialog(this._dialog); + + + this._elmts.dialogHeader.html($.i18n._('core-dialogs')["custom-tab-exp"]); + this._elmts.or_dialog_content.html($.i18n._('core-dialogs')["content"]); + this._elmts.or_dialog_download.html($.i18n._('core-dialogs')["download"]); + + this._elmts.selectAllButton.html($.i18n._('core-buttons')["select-all"]); + this._elmts.deselectAllButton.html($.i18n._('core-buttons')["deselect-all"]); + + this._elmts.or_dialog_outEmptyRow.html($.i18n._('core-dialogs')["out-empty-row"]); + + this._elmts.downloadPreviewButton.html($.i18n._('core-buttons')["preview"]); + this._elmts.downloadButton.html($.i18n._('core-buttons')["download"]); + + this._elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]); +// this._elmts.nextButton.html($.i18n._('core-buttons')["next"]); + + + this._elmts.tableNameLabel.html($.i18n._('core-dialogs')["tableNameLabel"]); + this._elmts.includeStructureLabel.html($.i18n._('core-dialogs')["for-include-structure-checkbox"]); + this._elmts.includeDropStatementLabel.html($.i18n._('core-dialogs')["for-include-drop-statement-checkbox"]); + this._elmts.includeContentLabel.html($.i18n._('core-dialogs')["for-include-content-checkbox"]); + + + $("#sql-exporter-tabs-content").css("display", ""); + $("#sql-exporter-tabs-download").css("display", ""); + $("#sql-exporter-tabs").tabs(); + + + /* + * Populate column list. + */ + for (var i = 0; i < theProject.columnModel.columns.length; i++) { + var column = theProject.columnModel.columns[i]; + var name = column.name; + var rowId = "sql-exporter-dialog-row" + i; + var selectBoxName = 'selectBoxRow' + i; + var sizeInputName = 'sizeInputRow' + i; + var applyAllBtnName = 'applyAllBtn' + i; + + var arr = [ + {val : 'VARCHAR', text: 'VARCHAR'}, + {val : 'TEXT', text: 'TEXT'}, + {val : 'INT', text: 'INT'}, + {val : 'NUMERIC', text: 'NUMERIC'}, + {val : 'CHAR', text: 'CHAR'}, + {val : 'DATE', text: 'DATE'}, + {val : 'TIMESTAMP', text: 'TIMESTAMP'} + ]; + + var sel = $('') + .attr('type', 'checkbox') + .attr('checked', 'checked') + .addClass("columnNameCheckboxStyle") + .appendTo(columnCell); + $('') + .text(name) + .appendTo(columnCell); + + var typeCell = $('') + .attr('width', '150px') + .appendTo(row); + sel.appendTo(typeCell); + + var sizeCell = $('') + .attr('width', '20px') + .appendTo(row); + $('') + .attr('type', 'text') + .attr('size', '8px') + .attr('id', sizeInputName) + .addClass("sql-exporter-dialog-input") + .appendTo(sizeCell); + + var applyAllCell = $('') + .attr('width', '60px') + .appendTo(row); + $('') + .attr('type', 'button') + .attr('value', 'Apply All') + .attr('id', applyAllBtnName) + .attr("rowIndex", i) + .appendTo(applyAllCell); + + $('#' + applyAllBtnName).on('click', function() { + var rowIndex = this.getAttribute('rowIndex'); + var typeValue = $("#selectBoxRow" + rowIndex).val(); + var sizeValue = $("#sizeInputRow" + rowIndex).val(); + + $('select.typeSelectClass').each(function() { + //alert("Value:" + this.value + " RowIndex:" + rowIndex + " TypeValue:" + typeValue + "" + this.value); + var rowId = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + if(rowIndex !== rowId){ + $("#" + id).val(typeValue); + } + + }); + $('input.sql-exporter-dialog-input').each(function() { + //alert("Value:" + this.value + " RowIndex:" + rowIndex + " TypeValue:" + typeValue + "" + this.value); + var rowId = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + if(rowIndex !== rowId){ + $("#" + id).val(sizeValue); + } + + + }); + + }); + + this._columnOptionMap[name] = { + name: name, + type: '', + size: '' + + }; + } + + + this._elmts.selectAllButton.click(function() { + $("input:checkbox[class=columnNameCheckboxStyle]").each(function () { + $(this).attr('checked', true) + }); + self._updateOptionCode(); + }); + this._elmts.deselectAllButton.click(function() { + $("input:checkbox[class=columnNameCheckboxStyle]").each(function () { + $(this).attr('checked', false) + }); + self._updateOptionCode(); + }); + + this._elmts.includeStructureCheckbox.click(function() { + var checked = $(this).is(':checked'); + //alert('checked ' + checked); + if(checked == true){ + $('#includeDropStatementCheckboxId').removeAttr("disabled"); + }else{ + $('#includeDropStatementCheckboxId').attr("disabled", true); + } + }); + + + this._elmts.cancelButton.click(function() { self._dismiss(); }); + this._elmts.downloadButton.click(function() { self._download(); }); + this._elmts.downloadPreviewButton.click(function(evt) { self._previewDownload(); }); + + this._configureUIFromOptionCode(options); + this._updateOptionCode(); + }; + + SqlExporterDialog.prototype._configureUIFromOptionCode = function(options) { + + this._elmts.tableNameTextBox.val(theProject.metadata.name); + this._elmts.outputEmptyRowsCheckbox.attr('checked', 'checked'); + + }; + + SqlExporterDialog.prototype._dismiss = function() { + DialogSystem.dismissUntil(this._level - 1); + }; + + SqlExporterDialog.prototype._previewDownload = function() { + this._postExport(true); + }; + + SqlExporterDialog.prototype._download = function() { + var result = this._postExport(false); + // alert("result::" + result); + if(result == true){ + this._dismiss(); + } + + }; + + SqlExporterDialog.prototype._postExport = function(preview) { + // var exportAllRowsCheckbox = this._elmts.exportAllRowsCheckbox[0].checked; + var options = this._getOptionCode(); + + if(options.columns == null || options.columns.length == 0){ + alert("Please select at least one column..."); + return false; + } + + var format = options.format; + var encoding = options.encoding; + + delete options.format; + delete options.encoding; + if (preview) { + options.limit = 10; + } + + // var ext = SqlExporterDialog.formats[format].extension; + var form = this._prepareSqlExportRowsForm(format, false, "sql"); + $('') + .attr("name", "options") + .attr("value", JSON.stringify(options)) + .appendTo(form); + if (encoding) { + $('') + .attr("name", "encoding") + .attr("value", encoding) + .appendTo(form); + } + if (!preview) { + $('') + .attr("name", "contentType") + .attr("value", "application/x-unknown") // force download + .appendTo(form); + } + + // alert("form::" + form); + document.body.appendChild(form); + + window.open("about:blank", "refine-export"); + form.submit(); + + document.body.removeChild(form); + return true; + }; + + SqlExporterDialog.prototype._prepareSqlExportRowsForm = function(format, includeEngine, ext) { + var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "post") + .attr("action", "command/core/export-rows/" + name + ((ext) ? ("." + ext) : "")) + .attr("target", "refine-export"); + + $('') + .attr("name", "project") + .attr("value", theProject.id) + .appendTo(form); + $('') + .attr("name", "format") + .attr("value", format) + .appendTo(form); + if (includeEngine) { + $('') + .attr("name", "engine") + .attr("value", JSON.stringify(ui.browsingEngine.getJSON())) + .appendTo(form); + } + + return form; + }; + + SqlExporterDialog.prototype._selectColumn = function(columnName) { + + this._elmts.columnNameSpan.text(columnName); + var columnOptions = this._columnOptionMap[columnName]; + alert("in _selectColumn:column type::" + columnOptions.type); + + }; + + SqlExporterDialog.prototype._updateCurrentColumnOptions = function() { +// var selectedColumnName = this._elmts.columnList.find('.sql-exporter-dialog-column.selected').attr('column'); +// //alert("_updateCurrentColumnOptions::" + selectedColumnName); +// var columnOptions = this._columnOptionMap[selectedColumnName]; +// columnOptions.type= this._elmts.columnOptionPane.find('input[name="sql-exporter-type"]:checked').val(); + + }; + + SqlExporterDialog.prototype._updateOptionCode = function() { + this._elmts.optionCodeInput.val(JSON.stringify(this._getOptionCode(), null, 2)); + }; + + + SqlExporterDialog.prototype._getOptionCode = function() { + var options = { + //format: this._dialog.find('input[name="sql-exporter-download-format"]:checked').val() + }; + var unescapeJavascriptString = function(s) { + try { + return JSON.parse('"' + s + '"'); + } catch (e) { + // We're not handling the case where the user doesn't escape double quotation marks. + return s; + } + }; + + options.format = 'sql'; + options.separator = ';'; + options.encoding = 'UTF-8'; + + options.outputBlankRows = this._elmts.outputEmptyRowsCheckbox[0].checked; + options.includeStructure = this._elmts.includeStructureCheckbox[0].checked; + options.includeDropStatement = this._elmts.includeDropStatementCheckbox[0].checked; + options.includeContent = this._elmts.includeContentCheckbox[0].checked; + options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '-'); + + + options.columns = []; + + var self = this; + this._elmts.columnListTable.find('.sql-exporter-dialog-row').each(function() { + if ($(this).find('input[type="checkbox"]')[0].checked) { + var name = this.getAttribute('column'); + var rowIndex = this.getAttribute('rowIndex'); + // alert("column::"+ name + " rowIndex::" + rowIndex); + + var selectedValue = $('#selectBoxRow' + rowIndex).val(); + //alert("selectedValue::"+ selectedValue); + var typeSize = 0; + if(selectedValue == 'VARCHAR' || selectedValue == 'CHAR' || selectedValue == 'INT' || selectedValue == 'NUMERIC'){ + typeSize = $('#sizeInputRow' + rowIndex).val(); + // alert("typeSize::" + typeSize); + } + + var fullColumnOptions = self._columnOptionMap[name]; + var columnOptions = { + name: name, + type: selectedValue, + size: typeSize + + }; + + // alert('checked type ' + columnIndex + ' =' + check_value); + + options.columns.push(columnOptions); + } + }); + //alert('options:' + options); + return options; + }; + \ No newline at end of file diff --git a/main/webapp/modules/core/scripts/project/exporters.js b/main/webapp/modules/core/scripts/project/exporters.js index 13eba106c..f6e5af26e 100644 --- a/main/webapp/modules/core/scripts/project/exporters.js +++ b/main/webapp/modules/core/scripts/project/exporters.js @@ -97,6 +97,11 @@ ExporterManager.MenuItems = [ "label": $.i18n._('core-project')["custom-tabular"], "click": function() { new CustomTabularExporterDialog(); } }, + { + "id" : "core/export-sql", + "label": $.i18n._('core-project')["sql-export"], + "click": function() { new SqlExporterDialog(); } + }, { "id" : "core/export-templating", "label": $.i18n._('core-project')["templating"], @@ -142,6 +147,7 @@ ExporterManager.handlers.exportRows = function(format, ext) { ExporterManager.prepareExportRowsForm = function(format, includeEngine, ext) { var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + //alert("name:" + name); var form = document.createElement("form"); $(form) .css("display", "none") diff --git a/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less new file mode 100644 index 000000000..4915575be --- /dev/null +++ b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less @@ -0,0 +1,94 @@ +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +@import-less url("../theme.less"); + +#sql-exporter-tabs .ui-tabs-panel { + padding: @padding_looser; + } + +.sql-exporter-tabs-content-main { + +} +.sql-exporter-columns, .sql-exporter-column-options { + /*border: 1px solid @chrome_primary;*/ + height: 8em; + padding: @padding_loose; + } +.sql-exporter-columns { + min-height: 200px; + overflow: auto; + } +.sql-exporter-selected-column { + font-weight: bold; + border: 1px solid @chrome_primary; + background: @chrome_primary; + padding: @padding_tighter; + .rounded_corners(); + } + +.sql-exporter-dialog-column { + /* border: 1px solid @chrome_primary;*/ + background: @fill_secondary; + padding: @padding_tighter; + margin: @padding_tight; + cursor: move; + .rounded_corners(); + } +.sql-exporter-dialog-column.selected { + background: @chrome_primary; + font-weight: bold; + } + +textarea.sql-exporter-code { + width: 100%; + border: 1px solid @chrome_primary; + height: 25em; + } +.sql-exporter-dialog-row { + margin-top: 2px; + padding-top: 2px; + margin-bottom: 2px; + margin-top: 2px; +} +tr.sql-exporter-dialog-row td { + border:1px dashed lightblue; + margin: 2px; +} +.sql-exporter-dialog-input { + margin:2px; + border: solid 2px lightblue; +} +.typeSelectClass { + margin-left:2px; +} \ No newline at end of file From 223d16db390f42aeff3de8861870fd9729c3ce8a Mon Sep 17 00:00:00 2001 From: TonyO Date: Sun, 25 Mar 2018 21:01:53 -0500 Subject: [PATCH 2/6] added ignore facet and trim column name options --- .../exporters/sql/SqlCreateBuilder.java | 12 +++- .../modules/core/langs/translation-en.json | 6 +- .../scripts/dialogs/sql-exporter-dialog.html | 12 +++- .../scripts/dialogs/sql-exporter-dialog.js | 60 ++++++++++++------- 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java index 588b208ba..32e043c88 100755 --- a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -59,6 +59,9 @@ public class SqlCreateBuilder { StringBuffer createSB = new StringBuffer(); JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); + + final boolean trimColNames = options == null ? true + : JSONUtilities.getBoolean(options, "trimColumnNames", false); int count = columnOptionArray.length(); @@ -68,9 +71,14 @@ public class SqlCreateBuilder { String name = JSONUtilities.getString(columnOptions, "name", null); String type = JSONUtilities.getString(columnOptions, "type", "VARCHAR"); String size = JSONUtilities.getString(columnOptions, "size", ""); + if (name != null) { - createSB.append(name + " "); - + if(trimColNames) { + createSB.append(name.trim() + " "); + }else{ + createSB.append(name + " "); + } + if (type.equals("VARCHAR")) { if (size.isEmpty()) { size = "255"; diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index be9ca529a..b727fe5fa 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -278,7 +278,11 @@ "for-include-structure-checkbox": "Include Schema", "for-include-drop-statement-checkbox": "Include Drop Statement", "for-include-content-checkbox": "Include Content", - "tableNameLabel": "Table Name:" + "tableNameLabel": "Table Name:", + "sqlExporterTrimColumns": "Trim Column Names", + "sqlExporterIgnoreFacets": "Ignore facets and filters and export all rows", + "sqlExporterOutputEmptyRows":"Output empty row (i.e. all cells null)" + }, "core-facets": { diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html index 136ef7a53..b76756c3b 100755 --- a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html @@ -51,9 +51,15 @@
- - - + + + + + + + + +
diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js index 411003f9f..ec6ffdea8 100755 --- a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js @@ -57,7 +57,7 @@ function SqlExporterDialog(options) { this._elmts.selectAllButton.html($.i18n._('core-buttons')["select-all"]); this._elmts.deselectAllButton.html($.i18n._('core-buttons')["deselect-all"]); - this._elmts.or_dialog_outEmptyRow.html($.i18n._('core-dialogs')["out-empty-row"]); + this._elmts.downloadPreviewButton.html($.i18n._('core-buttons')["preview"]); this._elmts.downloadButton.html($.i18n._('core-buttons')["download"]); @@ -70,7 +70,13 @@ function SqlExporterDialog(options) { this._elmts.includeStructureLabel.html($.i18n._('core-dialogs')["for-include-structure-checkbox"]); this._elmts.includeDropStatementLabel.html($.i18n._('core-dialogs')["for-include-drop-statement-checkbox"]); this._elmts.includeContentLabel.html($.i18n._('core-dialogs')["for-include-content-checkbox"]); - + + + this._elmts.sqlExportIgnoreFacetsLabel.html($.i18n._('core-dialogs')["sqlExporterIgnoreFacets"]); + this._elmts.sqlExportTrimAllColumnsLabel.html($.i18n._('core-dialogs')["sqlExporterTrimColumns"]); + this._elmts.sqlExportOutputEmptyRowsLabel.html($.i18n._('core-dialogs')["sqlExporterOutputEmptyRows"]); + + $("#sql-exporter-tabs-content").css("display", ""); $("#sql-exporter-tabs-download").css("display", ""); @@ -88,20 +94,29 @@ function SqlExporterDialog(options) { var sizeInputName = 'sizeInputRow' + i; var applyAllBtnName = 'applyAllBtn' + i; - var arr = [ - {val : 'VARCHAR', text: 'VARCHAR'}, - {val : 'TEXT', text: 'TEXT'}, - {val : 'INT', text: 'INT'}, - {val : 'NUMERIC', text: 'NUMERIC'}, - {val : 'CHAR', text: 'CHAR'}, - {val : 'DATE', text: 'DATE'}, - {val : 'TIMESTAMP', text: 'TIMESTAMP'} - ]; +// var arr = [ +// {val : 'VARCHAR', text: 'VARCHAR'}, +// {val : 'TEXT', text: 'TEXT'}, +// {val : 'INT', text: 'INT'}, +// {val : 'NUMERIC', text: 'NUMERIC'}, +// {val : 'CHAR', text: 'CHAR'}, +// {val : 'DATE', text: 'DATE'}, +// {val : 'TIMESTAMP', text: 'TIMESTAMP'} +// ]; var sel = $('') .attr("name", "options") .attr("value", JSON.stringify(options)) @@ -365,11 +382,12 @@ function SqlExporterDialog(options) { options.separator = ';'; options.encoding = 'UTF-8'; - options.outputBlankRows = this._elmts.outputEmptyRowsCheckbox[0].checked; + options.outputBlankRows = this._elmts.sqlExportOutputEmptyRowsCheckbox[0].checked; options.includeStructure = this._elmts.includeStructureCheckbox[0].checked; options.includeDropStatement = this._elmts.includeDropStatementCheckbox[0].checked; options.includeContent = this._elmts.includeContentCheckbox[0].checked; options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '-'); + options.trimColumnNames = this._elmts.sqlExportTrimAllColumnsCheckbox[0].checked; options.columns = []; From 3ffbcc9db7e98cdb32cab4d369fd19f121e9a067 Mon Sep 17 00:00:00 2001 From: TonyO Date: Sun, 25 Mar 2018 23:23:11 -0500 Subject: [PATCH 3/6] remove whitespaces between column names options --- .../exporters/sql/SqlCreateBuilder.java | 21 ++++++++++++++----- .../exporters/sql/SqlInsertBuilder.java | 12 ++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java index 32e043c88..ba0ffdadf 100755 --- a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -60,9 +60,9 @@ public class SqlCreateBuilder { JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); - final boolean trimColNames = options == null ? true - : JSONUtilities.getBoolean(options, "trimColumnNames", false); - + final boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + + //logger.info("Trim Column Names::" + trimColNames); int count = columnOptionArray.length(); for (int i = 0; i < count; i++) { @@ -71,10 +71,13 @@ public class SqlCreateBuilder { String name = JSONUtilities.getString(columnOptions, "name", null); String type = JSONUtilities.getString(columnOptions, "type", "VARCHAR"); String size = JSONUtilities.getString(columnOptions, "size", ""); - + //logger.info("Before Trim Column Names::" + name); + if (name != null) { if(trimColNames) { - createSB.append(name.trim() + " "); + String trimmedCol = name.replaceAll("\\s", ""); + createSB.append( trimmedCol + " "); + //logger.info("After Trim Column Names::" + name); }else{ createSB.append(name + " "); } @@ -134,5 +137,13 @@ public class SqlCreateBuilder { } return createSQL; } + +// public static void main(String[] args) { +// String column = "Column 1 With Spaces"; +// String newCol = column.replaceAll("\\s", ""); +// +// logger.info("Column after trim:" + newCol); +// +// } } diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java index 7e9e77a1b..97fea4077 100755 --- a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -85,9 +85,15 @@ public class SqlInsertBuilder { colOptionsMap.put("" + json.get("name"), json); }); } - - String colNamesWithSep = columns.stream() - .collect(Collectors.joining(",")); + final boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + String colNamesWithSep = null; + if(trimColNames) { + colNamesWithSep = columns.stream().map(col -> col.replaceAll("\\s", "")).collect(Collectors.joining(",")); + }else { + colNamesWithSep = columns.stream().collect(Collectors.joining(",")); + } + + StringBuffer values = new StringBuffer(); int idx = 0; From 000be9a7831efa5c95493b33562c451249260bc6 Mon Sep 17 00:00:00 2001 From: TonyO Date: Mon, 26 Mar 2018 17:33:26 -0500 Subject: [PATCH 4/6] removed unused code from Java classes --- .../refine/exporters/sql/SqlCreateBuilder.java | 16 ++++++---------- .../google/refine/exporters/sql/SqlExporter.java | 4 +++- .../refine/exporters/sql/SqlInsertBuilder.java | 7 ++++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java index ba0ffdadf..416b26e30 100755 --- a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -44,7 +44,7 @@ public class SqlCreateBuilder { private String table; - @SuppressWarnings("unused") + private List columns; private JSONObject options; @@ -56,13 +56,15 @@ public class SqlCreateBuilder { } public String getCreateSQL() { + if(logger.isDebugEnabled()) { + logger.debug("Create SQL with columns: {}", columns); + } StringBuffer createSB = new StringBuffer(); JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); final boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); - - //logger.info("Trim Column Names::" + trimColNames); + int count = columnOptionArray.length(); for (int i = 0; i < count; i++) { @@ -138,12 +140,6 @@ public class SqlCreateBuilder { return createSQL; } -// public static void main(String[] args) { -// String column = "Column 1 With Spaces"; -// String newCol = column.replaceAll("\\s", ""); -// -// logger.info("Column after trim:" + newCol); -// -// } + } diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java index 67fa176aa..d0629763d 100755 --- a/main/src/com/google/refine/exporters/sql/SqlExporter.java +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -66,7 +66,9 @@ public class SqlExporter implements WriterExporter { @Override public void export(final Project project, Properties params, Engine engine, final Writer writer) throws IOException { - + if(logger.isDebugEnabled()) { + logger.debug("export sql with params: {}", params); + } TabularSerializer serializer = new TabularSerializer() { @Override diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java index 97fea4077..9c60334d9 100755 --- a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -73,7 +73,9 @@ public class SqlInsertBuilder { * @return */ public String getInsertSQL() { - // logger.info("options values::::{}", options); + if(logger.isDebugEnabled()) { + logger.debug("Insert SQL with columns: {}", columns); + } JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); //logger.info("columnOptionArray::::{}", columnOptionArray); @@ -130,8 +132,7 @@ public class SqlInsertBuilder { values.append(","); } values.append("\n"); -// logger.info("running values:{}", values.toString()); - + } From c7c0d8884aac94fec21f5d410c641042285594bc Mon Sep 17 00:00:00 2001 From: TonyO Date: Sun, 22 Apr 2018 00:39:11 -0500 Subject: [PATCH 5/6] check empty cells --- .../exporting/SqlExporterCommand.java | 88 ++++++ .../exporters/sql/SqlCreateBuilder.java | 43 ++- .../google/refine/exporters/sql/SqlData.java | 20 +- .../refine/exporters/sql/SqlDataError.java | 34 +++ .../refine/exporters/sql/SqlExporter.java | 32 ++- .../exporters/sql/SqlInsertBuilder.java | 79 ++++-- .../webapp/modules/core/MOD-INF/controller.js | 2 + .../modules/core/langs/translation-en.json | 4 +- .../scripts/dialogs/sql-exporter-dialog.html | 27 +- .../scripts/dialogs/sql-exporter-dialog.js | 267 ++++++++++++------ .../styles/dialogs/sql-exporter-dialog.less | 7 + 11 files changed, 461 insertions(+), 142 deletions(-) create mode 100644 main/src/com/google/refine/commands/exporting/SqlExporterCommand.java create mode 100644 main/src/com/google/refine/exporters/sql/SqlDataError.java diff --git a/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java b/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java new file mode 100644 index 000000000..4503f5dc1 --- /dev/null +++ b/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java @@ -0,0 +1,88 @@ +package com.google.refine.commands.exporting; + +import java.io.IOException; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.ProjectManager; +import com.google.refine.commands.Command; +import com.google.refine.exporters.sql.SqlDataError; +import com.google.refine.exporters.sql.SqlExporter; +import com.google.refine.model.Project; + + +public class SqlExporterCommand extends Command { + + private static final Logger logger = LoggerFactory.getLogger("SqlExporterCommand"); + + @SuppressWarnings("unchecked") + public Properties getRequestParameters(HttpServletRequest request) { + Properties options = new Properties(); + Enumeration en = request.getParameterNames(); + while (en.hasMoreElements()) { + String name = en.nextElement(); + String value = request.getParameter(name); + options.put(name, value); + + } + return options; + } + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + ProjectManager.singleton.setBusy(true); + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + Project project = getProject(request); + //Engine engine = getEngine(request, project); + Properties params = getRequestParameters(request); + SqlExporter sqlExporter = new SqlExporter(); + List result = sqlExporter.validateSqlData(project, params); + if(result == null || result.isEmpty()) { + writer.key("code"); + writer.value("OK"); + }else { + writer.key("code"); + writer.value("Error"); + + writer.key("SqlDataErrors"); + writer.array(); + result.stream().forEach(err -> { + writer.object(); + + writer.key("errorcode"); + writer.value(err.getErrorCode()); + writer.key("errormessage"); + writer.value(err.getErrorMessage()); + + writer.endObject(); + + }); + writer.endArray(); + + } + + writer.endObject(); + + } catch (Exception e) { + throw new ServletException(e); + } finally { + ProjectManager.singleton.setBusy(false); + } + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java index 416b26e30..e6450df55 100755 --- a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -62,8 +62,9 @@ public class SqlCreateBuilder { StringBuffer createSB = new StringBuffer(); JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); + boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + - final boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); int count = columnOptionArray.length(); @@ -71,39 +72,45 @@ public class SqlCreateBuilder { JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i); if (columnOptions != null) { String name = JSONUtilities.getString(columnOptions, "name", null); - String type = JSONUtilities.getString(columnOptions, "type", "VARCHAR"); + String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR); String size = JSONUtilities.getString(columnOptions, "size", ""); - //logger.info("Before Trim Column Names::" + name); + boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true); + String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null); + + String allowNullStr = "NULL"; + if(!allowNull) { + allowNullStr = "NOT NULL"; + } + if (name != null) { if(trimColNames) { String trimmedCol = name.replaceAll("\\s", ""); createSB.append( trimmedCol + " "); - //logger.info("After Trim Column Names::" + name); }else{ createSB.append(name + " "); } - if (type.equals("VARCHAR")) { + if (type.equals(SqlData.SQL_TYPE_VARCHAR)) { if (size.isEmpty()) { size = "255"; } createSB.append(type + "(" + size + ")"); - } else if (type.equals("CHAR")) { + } else if (type.equals(SqlData.SQL_TYPE_CHAR)) { if (size.isEmpty()) { size = "10"; } createSB.append(type + "(" + size + ")"); - } else if (type.equals("INT") || type.equals("INTEGER")) { + } else if (type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER)) { if (size.isEmpty()) { createSB.append(type); } else { createSB.append(type + "(" + size + ")"); } - } else if (type.equals("NUMERIC")) { + } else if (type.equals(SqlData.SQL_TYPE_NUMERIC)) { if (size.isEmpty()) { createSB.append(type); } else { @@ -112,7 +119,17 @@ public class SqlCreateBuilder { } else { createSB.append(type); } - + + createSB.append(" " + allowNullStr); + if(defaultValue != null && !defaultValue.isEmpty()) { + if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { + createSB.append(" DEFAULT " + "'" + defaultValue + "'"); + }else { + createSB.append(" DEFAULT " + defaultValue); + } + + } + if (i < count - 1) { createSB.append(","); } @@ -124,8 +141,14 @@ public class SqlCreateBuilder { StringBuffer sql = new StringBuffer(); boolean includeDrop = JSONUtilities.getBoolean(options, "includeDropStatement", false); + boolean addIfExist = options == null ? false : JSONUtilities.getBoolean(options, "includeIfExistWithDropStatement", true); if (includeDrop) { - sql.append("DROP TABLE " + table + ";\n"); + if(addIfExist) { + sql.append("DROP TABLE IF EXISTS " + table + ";\n"); + }else { + sql.append("DROP TABLE " + table + ";\n"); + } + } sql.append("CREATE TABLE ").append(table); diff --git a/main/src/com/google/refine/exporters/sql/SqlData.java b/main/src/com/google/refine/exporters/sql/SqlData.java index 6e595b733..383c17e8d 100755 --- a/main/src/com/google/refine/exporters/sql/SqlData.java +++ b/main/src/com/google/refine/exporters/sql/SqlData.java @@ -32,9 +32,17 @@ package com.google.refine.exporters.sql; public class SqlData { - final public String columnName; - final public Object value; - final public String text; + private String columnName; + private Object value; + private String text; + public static final String SQL_TYPE_VARCHAR = "VARCHAR"; + public static final String SQL_TYPE_CHAR = "CHAR"; + public static final String SQL_TYPE_TEXT = "TEXT"; + public static final String SQL_TYPE_INTEGER = "INTEGER"; + public static final String SQL_TYPE_INT = "INT"; + public static final String SQL_TYPE_NUMERIC = "NUMERIC"; + + public SqlData(String columnName, Object value, String text) { this.columnName = columnName; @@ -58,4 +66,10 @@ public class SqlData { return text; } + + @Override + public String toString() { + return "SqlData [columnName=" + columnName + ", value=" + value + ", text=" + text + "]"; + } + } diff --git a/main/src/com/google/refine/exporters/sql/SqlDataError.java b/main/src/com/google/refine/exporters/sql/SqlDataError.java new file mode 100644 index 000000000..521443d87 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlDataError.java @@ -0,0 +1,34 @@ +package com.google.refine.exporters.sql; + + +public class SqlDataError { + private int errorCode; + private String errorMessage; + + public SqlDataError(int errorCode, String errorMessage) { + super(); + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + + public int getErrorCode() { + return errorCode; + } + + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + + public String getErrorMessage() { + return errorMessage; + } + + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java index d0629763d..f538c45d1 100755 --- a/main/src/com/google/refine/exporters/sql/SqlExporter.java +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -49,11 +49,14 @@ import com.google.refine.util.JSONUtilities; public class SqlExporter implements WriterExporter { + private static final Logger logger = LoggerFactory.getLogger("SqlExporter"); public static final String NO_COL_SELECTED_ERROR = "****NO COLUMNS SELECTED****"; public static final String NO_OPTIONS_PRESENT_ERROR = "****NO OPTIONS PRESENT****"; - - private final static Logger logger = LoggerFactory.getLogger("SqlExporter"); - + //JSON Property names + public static final String JSON_INCLUDE_STRUCTURE = "includeStructure"; + public static final String JSON_INCLUDE_CONTENT = "includeContent"; + public static final String JSON_TABLE_NAME = "tableName"; + private List columnNames = new ArrayList(); private List> sqlDataList = new ArrayList>(); private JSONObject sqlOptions; @@ -92,7 +95,7 @@ public class SqlExporter implements WriterExporter { } String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName(); - Object tableNameManual = sqlOptions.get("tableName"); + Object tableNameManual = sqlOptions.get(JSON_TABLE_NAME); if (tableNameManual != null && !tableNameManual.toString().isEmpty()) { tableName = tableNameManual.toString(); @@ -103,15 +106,14 @@ public class SqlExporter implements WriterExporter { sqlOptions); final boolean includeStructure = sqlOptions == null ? true - : JSONUtilities.getBoolean(sqlOptions, "includeStructure", true); + : JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_STRUCTURE, true); final boolean includeContent = sqlOptions == null ? true - : JSONUtilities.getBoolean(sqlOptions, "includeContent", true); + : JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_CONTENT, true); if (includeStructure) { String sqlCreateStr = createBuilder.getCreateSQL(); writer.write(sqlCreateStr); - } if (includeContent) { @@ -143,9 +145,12 @@ public class SqlExporter implements WriterExporter { ArrayList values = new ArrayList<>(); for (CellData cellData : cells) { - if (cellData != null && cellData.text != null) { - SqlData newSql = new SqlData(cellData.columnName, cellData.value, cellData.text); - values.add(newSql); + if (cellData != null) { + if(cellData.text == null || cellData.text.isEmpty()) { + values.add(new SqlData(cellData.columnName, "", "")); + }else { + values.add(new SqlData(cellData.columnName, cellData.value, cellData.text)); + } } @@ -158,4 +163,11 @@ public class SqlExporter implements WriterExporter { CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer); } + + public List validateSqlData(Project project, Properties params){ + + logger.info("Param Name:{}, Param Value:{}", project, params); + return null; + + } } diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java index 9c60334d9..efd7686b5 100755 --- a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -72,55 +72,77 @@ public class SqlInsertBuilder { * Get Insert Sql * @return */ - public String getInsertSQL() { + public String getInsertSQL(){ if(logger.isDebugEnabled()) { logger.debug("Insert SQL with columns: {}", columns); } - JSONArray columnOptionArray = options == null ? null : - JSONUtilities.getArray(options, "columns"); - //logger.info("columnOptionArray::::{}", columnOptionArray); + JSONArray colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); Map colOptionsMap = new HashMap(); - if(columnOptionArray != null) { - columnOptionArray.forEach(c -> { + if(colOptionArray != null) { + colOptionArray.forEach(c -> { JSONObject json = (JSONObject)c; colOptionsMap.put("" + json.get("name"), json); }); } - final boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); - String colNamesWithSep = null; - if(trimColNames) { - colNamesWithSep = columns.stream().map(col -> col.replaceAll("\\s", "")).collect(Collectors.joining(",")); - }else { - colNamesWithSep = columns.stream().collect(Collectors.joining(",")); - } - + boolean nullValueToEmptyStr = options == null ? false : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true); + StringBuffer values = new StringBuffer(); int idx = 0; - for(ArrayList sqlCellData : sqlDataList) { + for(ArrayList sqlRow : sqlDataList) { StringBuilder rowValue = new StringBuilder(); - //logger.info(" row.size:{}", row.size()); - for(SqlData val : sqlCellData) { - + + for(SqlData val : sqlRow) { + JSONObject jsonOb = colOptionsMap.get(val.getColumnName()); String type = (String)jsonOb.get("type"); + String defaultValue = (String)jsonOb.get("defaultValue"); + boolean allowNullChkBox = (boolean)jsonOb.get("allowNull"); + + if(type == null) { - type = "VARCHAR"; + type = SqlData.SQL_TYPE_VARCHAR; } - if(type.equals("VARCHAR") || type.equals("CHAR") || type.equals("TEXT")) { + //Character Types + if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { + + if(!allowNullChkBox) { + throw new RuntimeException("bad input"); + } + if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) { + // logger.info("Appending empty String:::{}" , val.getText()); + if(defaultValue != null && !defaultValue.isEmpty()) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append("null"); + } + + }else { + rowValue.append("'" + val.getText() + "'"); + } + + }else {//Numeric Types + + if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) { + // logger.info("Appending empty String others:::{}" , val.getText()); + if(defaultValue != null && !defaultValue.isEmpty()) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append("null"); + } + }else { + rowValue.append(val.getText()); + } - String value = "'" + val.text + "'"; - rowValue.append(value); - }else { - rowValue.append(val.text); } rowValue.append(","); - //logger.info("jsonObject:{}", jsonOb); - + } + + idx++; String rowValString = rowValue.toString(); rowValString = rowValString.substring(0, rowValString.length() - 1); @@ -135,6 +157,11 @@ public class SqlInsertBuilder { } + boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + String colNamesWithSep = columns.stream().map(col -> col.replaceAll("\\s", "")).collect(Collectors.joining(","));; + if(!trimColNames) { + colNamesWithSep = columns.stream().collect(Collectors.joining(",")); + } String valuesString = values.toString(); valuesString = valuesString.substring(0, valuesString.length() - 1); diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index a11a5f238..bffff591f 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -151,6 +151,8 @@ function registerCommands() { RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand()); RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand()); + + RS.registerCommand(module, "preview-sql-export", new Packages.com.google.refine.commands.exporting.SqlExporterCommand()); } function registerOperations() { diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index b727fe5fa..e95e48275 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -281,7 +281,9 @@ "tableNameLabel": "Table Name:", "sqlExporterTrimColumns": "Trim Column Names", "sqlExporterIgnoreFacets": "Ignore facets and filters and export all rows", - "sqlExporterOutputEmptyRows":"Output empty row (i.e. all cells null)" + "sqlExporterOutputEmptyRows":"Output empty row (i.e. all cells null)", + "for-include-if-exist-drop-stmt-checkbox": "Include 'If Exists' in Drop Statement", + "for-null-cell-value-to-empty-str-label": "Convert Null Value to null in Insert" }, diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html index b76756c3b..37c20803f 100755 --- a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html @@ -25,13 +25,11 @@ Field Name SQL Type Size + + Allow Null + Default - + @@ -75,12 +73,17 @@ + +
+ + + @@ -88,12 +91,24 @@ + + + + + + + + + + + +
diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js index ec6ffdea8..4e266004c 100755 --- a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js @@ -48,16 +48,12 @@ function SqlExporterDialog(options) { this._dialog = $(DOM.loadHTML("core", "scripts/dialogs/sql-exporter-dialog.html")); this._elmts = DOM.bind(this._dialog); this._level = DialogSystem.showDialog(this._dialog); - - this._elmts.dialogHeader.html($.i18n._('core-dialogs')["custom-tab-exp"]); this._elmts.or_dialog_content.html($.i18n._('core-dialogs')["content"]); this._elmts.or_dialog_download.html($.i18n._('core-dialogs')["download"]); this._elmts.selectAllButton.html($.i18n._('core-buttons')["select-all"]); this._elmts.deselectAllButton.html($.i18n._('core-buttons')["deselect-all"]); - - this._elmts.downloadPreviewButton.html($.i18n._('core-buttons')["preview"]); this._elmts.downloadButton.html($.i18n._('core-buttons')["download"]); @@ -65,44 +61,33 @@ function SqlExporterDialog(options) { this._elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]); // this._elmts.nextButton.html($.i18n._('core-buttons')["next"]); - this._elmts.tableNameLabel.html($.i18n._('core-dialogs')["tableNameLabel"]); this._elmts.includeStructureLabel.html($.i18n._('core-dialogs')["for-include-structure-checkbox"]); this._elmts.includeDropStatementLabel.html($.i18n._('core-dialogs')["for-include-drop-statement-checkbox"]); this._elmts.includeContentLabel.html($.i18n._('core-dialogs')["for-include-content-checkbox"]); + this._elmts.includeIfExistDropStatementLabel.html($.i18n._('core-dialogs')["for-include-if-exist-drop-stmt-checkbox"]); - + this._elmts.nullCellValueToEmptyStringLabel.html($.i18n._('core-dialogs')["for-null-cell-value-to-empty-str-label"]); + this._elmts.sqlExportIgnoreFacetsLabel.html($.i18n._('core-dialogs')["sqlExporterIgnoreFacets"]); this._elmts.sqlExportTrimAllColumnsLabel.html($.i18n._('core-dialogs')["sqlExporterTrimColumns"]); this._elmts.sqlExportOutputEmptyRowsLabel.html($.i18n._('core-dialogs')["sqlExporterOutputEmptyRows"]); - - $("#sql-exporter-tabs-content").css("display", ""); $("#sql-exporter-tabs-download").css("display", ""); $("#sql-exporter-tabs").tabs(); - - - /* - * Populate column list. - */ + for (var i = 0; i < theProject.columnModel.columns.length; i++) { - var column = theProject.columnModel.columns[i]; - var name = column.name; - var rowId = "sql-exporter-dialog-row" + i; - var selectBoxName = 'selectBoxRow' + i; - var sizeInputName = 'sizeInputRow' + i; - var applyAllBtnName = 'applyAllBtn' + i; + var column = theProject.columnModel.columns[i]; + var name = column.name; + var rowId = "sql-exporter-dialog-row" + i; + var selectBoxName = 'selectBoxRow' + i; + var sizeInputName = 'sizeInputRow' + i; + var applyAllBtnName = 'applyAllBtn' + i; -// var arr = [ -// {val : 'VARCHAR', text: 'VARCHAR'}, -// {val : 'TEXT', text: 'TEXT'}, -// {val : 'INT', text: 'INT'}, -// {val : 'NUMERIC', text: 'NUMERIC'}, -// {val : 'CHAR', text: 'CHAR'}, -// {val : 'DATE', text: 'DATE'}, -// {val : 'TIMESTAMP', text: 'TIMESTAMP'} -// ]; + var allowNullChkBoxName = 'allowNullChkBox' + i; + var defaultValueTextBoxName = 'defaultValueTextBox' + i; + var sel = $('') + //create and add Row + var row = $('') .addClass("sql-exporter-dialog-row") .attr('id', rowId) .attr("column", name) .attr("rowIndex", i) .appendTo(this._elmts.columnListTable); + //create and add column var columnCell = $('
') .attr('width', '150px') .appendTo(row); @@ -166,15 +148,40 @@ function SqlExporterDialog(options) { .addClass("sql-exporter-dialog-input") .appendTo(sizeCell); - var applyAllCell = $('') - .attr('width', '60px') - .appendTo(row); + var applyAllCell = $('') + .attr('width', '60px') + .appendTo(row); + $('') + .attr('type', 'button') + .attr('value', 'Apply All') + .attr('id', applyAllBtnName) + .attr("rowIndex", i) + .appendTo(applyAllCell); + + var nullableCell = $('') + .attr('width', '90px') + .addClass("allowNullCellStyle") + .appendTo(row); $('') - .attr('type', 'button') - .attr('value', 'Apply All') - .attr('id', applyAllBtnName) - .attr("rowIndex", i) - .appendTo(applyAllCell); + .attr('type', 'checkbox') + .attr('checked', 'checked') + .attr('id', allowNullChkBoxName) + .attr("rowIndex", i) + .addClass("allowNullCheckboxStyle") + .appendTo(nullableCell); + + var defValueCell = $('') + .attr('width', '30px') + .appendTo(row); + $('') + .attr('type', 'text') + .attr('size', '8px') + .attr('id', defaultValueTextBoxName) + .attr("rowIndex", i) + .addClass("defaultValueTextBoxStyle") + .appendTo(defValueCell); + + $('#' + applyAllBtnName).on('click', function() { var rowIndex = this.getAttribute('rowIndex'); @@ -191,18 +198,29 @@ function SqlExporterDialog(options) { }); $('input.sql-exporter-dialog-input').each(function() { - //alert("Value:" + this.value + " RowIndex:" + rowIndex + " TypeValue:" + typeValue + "" + this.value); var rowId = this.getAttribute('rowIndex'); var id = this.getAttribute('id'); if(rowIndex !== rowId){ $("#" + id).val(sizeValue); } - - + }); }); + $('#' + allowNullChkBoxName).on('click', function() { + var rowIndex = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + var checked = $(this).is(':checked');; + + if(checked == false){ + $('#defaultValueTextBox'+ rowIndex).prop("disabled", false); + }else{ + $('#defaultValueTextBox'+ rowIndex).val(""); + $('#defaultValueTextBox'+ rowIndex).prop("disabled", true); + } + }); + this._columnOptionMap[name] = { name: name, type: '', @@ -210,7 +228,27 @@ function SqlExporterDialog(options) { }; } - + + this._elmts.allowNullToggleCheckbox.click(function() { + var checked = $(this).is(':checked'); + if(checked == true){ + $("input:checkbox[class=allowNullCheckboxStyle]").each(function () { + $(this).attr('checked', true); + }); + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).attr('disabled', true); + }); + + }else{ + $("input:checkbox[class=allowNullCheckboxStyle]").each(function () { + $(this).attr('checked', false); + }); + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).attr('disabled', false); + }); + } + + }); this._elmts.selectAllButton.click(function() { $("input:checkbox[class=columnNameCheckboxStyle]").each(function () { @@ -230,8 +268,20 @@ function SqlExporterDialog(options) { //alert('checked ' + checked); if(checked == true){ $('#includeDropStatementCheckboxId').removeAttr("disabled"); + $('#includeIfExistDropStatementCheckboxId').removeAttr("disabled"); }else{ $('#includeDropStatementCheckboxId').attr("disabled", true); + $('#includeIfExistDropStatementCheckboxId').attr("disabled", true); + } + }); + + this._elmts.includeContentCheckbox.click(function() { + var checked = $(this).is(':checked'); + if(checked == true){ + $('#nullCellValueToEmptyStringCheckboxId').removeAttr("disabled"); + }else{ + $('#nullCellValueToEmptyStringCheckboxId').attr("disabled", true); + } }); @@ -249,8 +299,13 @@ function SqlExporterDialog(options) { this._elmts.tableNameTextBox.val(theProject.metadata.name); this._elmts.sqlExportOutputEmptyRowsCheckbox.attr('checked', 'checked'); this._elmts.sqlExportTrimAllColumnsCheckbox.attr('checked', 'checked'); + this._elmts.nullCellValueToEmptyStringLabel.attr('checked', 'checked'); - + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).prop("disabled", true); + }); + + }; SqlExporterDialog.prototype._dismiss = function() { @@ -271,6 +326,7 @@ function SqlExporterDialog(options) { }; SqlExporterDialog.prototype._postExport = function(preview) { + var self = this; var exportAllRowsCheckbox = this._elmts.sqlExportAllRowsCheckbox[0].checked; var options = this._getOptionCode(); @@ -279,42 +335,71 @@ function SqlExporterDialog(options) { return false; } - var format = options.format; - var encoding = options.encoding; + var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + var sqlExportInput = {}; + sqlExportInput.name = name; + sqlExportInput.options = JSON.stringify(options); + sqlExportInput.project = theProject.id; - delete options.format; - delete options.encoding; - if (preview) { - options.limit = 10; - } + //validate form @ backend + $.post( + "command/core/preview-sql-export", + sqlExportInput, + function(sqlValidationResult) { + if(sqlValidationResult && sqlValidationResult.code === 'OK'){ + //generate code + + var format = options.format; + var encoding = options.encoding; + + delete options.format; + delete options.encoding; + if (preview) { + options.limit = 10; + } + + // var ext = SqlExporterDialog.formats[format].extension; + var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql"); + $('') + .attr("name", "options") + .attr("value", JSON.stringify(options)) + .appendTo(form); + if (encoding) { + $('') + .attr("name", "encoding") + .attr("value", encoding) + .appendTo(form); + } + if (!preview) { + $('') + .attr("name", "contentType") + .attr("value", "application/x-unknown") // force download + .appendTo(form); + } + + // alert("form::" + form); + document.body.appendChild(form); + + window.open("about:blank", "refine-export"); + form.submit(); + + document.body.removeChild(form); + return true; + + }else{ + window.alert("Problem with sql code genration"); + return false; + } + + }, + "json" + ).fail(function( jqXhr, textStatus, errorThrown ){ + alert( textStatus + ':' + errorThrown ); + return false; + }); - // var ext = SqlExporterDialog.formats[format].extension; - var form = this._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql"); - $('') - .attr("name", "options") - .attr("value", JSON.stringify(options)) - .appendTo(form); - if (encoding) { - $('') - .attr("name", "encoding") - .attr("value", encoding) - .appendTo(form); - } - if (!preview) { - $('') - .attr("name", "contentType") - .attr("value", "application/x-unknown") // force download - .appendTo(form); - } - - // alert("form::" + form); - document.body.appendChild(form); + return true; - window.open("about:blank", "refine-export"); - form.submit(); - - document.body.removeChild(form); - return true; }; SqlExporterDialog.prototype._prepareSqlExportRowsForm = function(format, includeEngine, ext) { @@ -386,8 +471,11 @@ function SqlExporterDialog(options) { options.includeStructure = this._elmts.includeStructureCheckbox[0].checked; options.includeDropStatement = this._elmts.includeDropStatementCheckbox[0].checked; options.includeContent = this._elmts.includeContentCheckbox[0].checked; - options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '-'); + options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '_'); options.trimColumnNames = this._elmts.sqlExportTrimAllColumnsCheckbox[0].checked; + + options.convertNulltoEmptyString = this._elmts.nullCellValueToEmptyStringCheckbox[0].checked; + options.includeIfExistWithDropStatement = this._elmts.includeIfExistDropStatementCheckbox[0].checked; options.columns = []; @@ -397,21 +485,28 @@ function SqlExporterDialog(options) { if ($(this).find('input[type="checkbox"]')[0].checked) { var name = this.getAttribute('column'); var rowIndex = this.getAttribute('rowIndex'); - // alert("column::"+ name + " rowIndex::" + rowIndex); var selectedValue = $('#selectBoxRow' + rowIndex).val(); - //alert("selectedValue::"+ selectedValue); + var typeSize = 0; - if(selectedValue == 'VARCHAR' || selectedValue == 'CHAR' || selectedValue == 'INT' || selectedValue == 'NUMERIC'){ + if(selectedValue === 'VARCHAR' || selectedValue === 'CHAR' || selectedValue === 'INT' || selectedValue === 'NUMERIC'){ typeSize = $('#sizeInputRow' + rowIndex).val(); // alert("typeSize::" + typeSize); } + var allowNullChkBoxName = $('#allowNullChkBox' + rowIndex).is(':checked'); + var defaultValueTextBoxName = $('#defaultValueTextBox' + rowIndex).val(); + // alert("allowNullChkBoxName::" + allowNullChkBoxName); + // alert("defaultValueTextBoxName::" + defaultValueTextBoxName); + var fullColumnOptions = self._columnOptionMap[name]; var columnOptions = { name: name, type: selectedValue, - size: typeSize + size: typeSize, + allowNull: allowNullChkBoxName, + defaultValue: defaultValueTextBoxName, + nullValueToEmptyStr: options.convertNulltoEmptyString }; diff --git a/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less index 4915575be..c090bf1dd 100644 --- a/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less +++ b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less @@ -89,6 +89,13 @@ tr.sql-exporter-dialog-row td { margin:2px; border: solid 2px lightblue; } +.defaultValueTextBoxStyle { + margin:2px; +} +.allowNullCellStyle { + text-align: center; + vertical-align: middle; +} .typeSelectClass { margin-left:2px; } \ No newline at end of file From 0eb7b4082f018c9bc6e22f1e6008667dcd067aef Mon Sep 17 00:00:00 2001 From: TonyO Date: Thu, 26 Apr 2018 20:50:46 -0500 Subject: [PATCH 6/6] added null and default value options --- .../exporting/SqlExporterCommand.java | 88 ---------- .../commands/project/ExportRowsCommand.java | 16 +- .../CustomizableTabularExporterUtilities.java | 12 +- .../exporters/sql/SqlCreateBuilder.java | 11 +- .../google/refine/exporters/sql/SqlData.java | 2 + .../refine/exporters/sql/SqlDataError.java | 34 ---- .../refine/exporters/sql/SqlExporter.java | 19 +- .../exporters/sql/SqlExporterException.java | 16 ++ .../exporters/sql/SqlInsertBuilder.java | 120 ++++++++++--- .../tests/exporters/sql/SqlExporterTests.java | 163 +++++++++++++++++- .../webapp/modules/core/MOD-INF/controller.js | 3 +- .../scripts/dialogs/sql-exporter-dialog.js | 96 ++++------- 12 files changed, 348 insertions(+), 232 deletions(-) delete mode 100644 main/src/com/google/refine/commands/exporting/SqlExporterCommand.java delete mode 100644 main/src/com/google/refine/exporters/sql/SqlDataError.java create mode 100644 main/src/com/google/refine/exporters/sql/SqlExporterException.java diff --git a/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java b/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java deleted file mode 100644 index 4503f5dc1..000000000 --- a/main/src/com/google/refine/commands/exporting/SqlExporterCommand.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.google.refine.commands.exporting; - -import java.io.IOException; -import java.util.Enumeration; -import java.util.List; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.ProjectManager; -import com.google.refine.commands.Command; -import com.google.refine.exporters.sql.SqlDataError; -import com.google.refine.exporters.sql.SqlExporter; -import com.google.refine.model.Project; - - -public class SqlExporterCommand extends Command { - - private static final Logger logger = LoggerFactory.getLogger("SqlExporterCommand"); - - @SuppressWarnings("unchecked") - public Properties getRequestParameters(HttpServletRequest request) { - Properties options = new Properties(); - Enumeration en = request.getParameterNames(); - while (en.hasMoreElements()) { - String name = en.nextElement(); - String value = request.getParameter(name); - options.put(name, value); - - } - return options; - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - ProjectManager.singleton.setBusy(true); - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - Project project = getProject(request); - //Engine engine = getEngine(request, project); - Properties params = getRequestParameters(request); - SqlExporter sqlExporter = new SqlExporter(); - List result = sqlExporter.validateSqlData(project, params); - if(result == null || result.isEmpty()) { - writer.key("code"); - writer.value("OK"); - }else { - writer.key("code"); - writer.value("Error"); - - writer.key("SqlDataErrors"); - writer.array(); - result.stream().forEach(err -> { - writer.object(); - - writer.key("errorcode"); - writer.value(err.getErrorCode()); - writer.key("errormessage"); - writer.value(err.getErrorMessage()); - - writer.endObject(); - - }); - writer.endArray(); - - } - - writer.endObject(); - - } catch (Exception e) { - throw new ServletException(e); - } finally { - ProjectManager.singleton.setBusy(false); - } - } - -} diff --git a/main/src/com/google/refine/commands/project/ExportRowsCommand.java b/main/src/com/google/refine/commands/project/ExportRowsCommand.java index 664064b19..3270c671c 100644 --- a/main/src/com/google/refine/commands/project/ExportRowsCommand.java +++ b/main/src/com/google/refine/commands/project/ExportRowsCommand.java @@ -44,6 +44,10 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.commands.Command; @@ -52,9 +56,12 @@ import com.google.refine.exporters.Exporter; import com.google.refine.exporters.ExporterRegistry; import com.google.refine.exporters.StreamExporter; import com.google.refine.exporters.WriterExporter; +import com.google.refine.exporters.sql.SqlExporterException; + import com.google.refine.model.Project; public class ExportRowsCommand extends Command { + private static final Logger logger = LoggerFactory.getLogger("ExportRowsCommand"); @SuppressWarnings("unchecked") static public Properties getRequestParameters(HttpServletRequest request) { @@ -73,6 +80,7 @@ public class ExportRowsCommand extends Command { throws ServletException, IOException { ProjectManager.singleton.setBusy(true); + try { Project project = getProject(request); Engine engine = getEngine(request, project); @@ -100,7 +108,8 @@ public class ExportRowsCommand extends Command { ((WriterExporter) exporter).export(project, params, engine, writer); writer.close(); - } else if (exporter instanceof StreamExporter) { + } + else if (exporter instanceof StreamExporter) { response.setCharacterEncoding("UTF-8"); OutputStream stream = response.getOutputStream(); @@ -108,12 +117,17 @@ public class ExportRowsCommand extends Command { stream.close(); // } else if (exporter instanceof UrlExporter) { // ((UrlExporter) exporter).export(project, options, engine); + } else { // TODO: Should this use ServletException instead of respondException? respondException(response, new RuntimeException("Unknown exporter type")); } } catch (Exception e) { // Use generic error handling rather than our JSON handling + logger.info("error:{}", e.getMessage()); + if (e instanceof SqlExporterException) { + response.sendError(HttpStatus.SC_BAD_REQUEST, e.getMessage()); + } throw new ServletException(e); } finally { ProjectManager.singleton.setBusy(false); diff --git a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java index 7f392fc46..82dd19279 100644 --- a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java +++ b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java @@ -229,12 +229,16 @@ abstract public class CustomizableTabularExporterUtilities { Map identifierSpaceToUrl = null; + //SQLExporter parameter to convert null cell value to empty string + boolean includeNullFieldValue = false; + CellFormatter() { dateFormatter = new SimpleDateFormat(fullIso8601); } CellFormatter(JSONObject options) { JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings"); + includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false); if (reconSettings != null) { String reconOutputString = JSONUtilities.getString(reconSettings, "output", null); if ("entity-name".equals(reconOutputString)) { @@ -354,6 +358,12 @@ abstract public class CustomizableTabularExporterUtilities { } return new CellData(column.getName(), value, text, link); } + }else {//added for sql exporter + + if(includeNullFieldValue) { + return new CellData(column.getName(), "", "", ""); + } + } return null; } @@ -384,4 +394,4 @@ abstract public class CustomizableTabularExporterUtilities { } } } -} +} \ No newline at end of file diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java index e6450df55..55a62b591 100755 --- a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -43,16 +43,15 @@ public class SqlCreateBuilder { private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder"); private String table; - - private List columns; - private JSONObject options; + public SqlCreateBuilder(String table, List columns, JSONObject options) { this.table = table; this.columns = columns; this.options = options; + } public String getCreateSQL() { @@ -76,6 +75,7 @@ public class SqlCreateBuilder { String size = JSONUtilities.getString(columnOptions, "size", ""); boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true); String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null); + logger.info("allowNull::{}" , allowNull); String allowNullStr = "NULL"; if(!allowNull) { @@ -125,6 +125,11 @@ public class SqlCreateBuilder { if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { createSB.append(" DEFAULT " + "'" + defaultValue + "'"); }else { + try { + Integer.parseInt(defaultValue); + }catch(NumberFormatException nfe) { + throw new SqlExporterException(defaultValue + " is not compatible with column type :" + type); + } createSB.append(" DEFAULT " + defaultValue); } diff --git a/main/src/com/google/refine/exporters/sql/SqlData.java b/main/src/com/google/refine/exporters/sql/SqlData.java index 383c17e8d..b72921701 100755 --- a/main/src/com/google/refine/exporters/sql/SqlData.java +++ b/main/src/com/google/refine/exporters/sql/SqlData.java @@ -41,6 +41,8 @@ public class SqlData { public static final String SQL_TYPE_INTEGER = "INTEGER"; public static final String SQL_TYPE_INT = "INT"; public static final String SQL_TYPE_NUMERIC = "NUMERIC"; + public static final String SQL_TYPE_DATE = "DATE"; + public static final String SQL_TYPE_TIMESTAMP = "TIMESTAMP"; diff --git a/main/src/com/google/refine/exporters/sql/SqlDataError.java b/main/src/com/google/refine/exporters/sql/SqlDataError.java deleted file mode 100644 index 521443d87..000000000 --- a/main/src/com/google/refine/exporters/sql/SqlDataError.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.google.refine.exporters.sql; - - -public class SqlDataError { - private int errorCode; - private String errorMessage; - - public SqlDataError(int errorCode, String errorMessage) { - super(); - this.errorCode = errorCode; - this.errorMessage = errorMessage; - } - - - public int getErrorCode() { - return errorCode; - } - - - public void setErrorCode(int errorCode) { - this.errorCode = errorCode; - } - - - public String getErrorMessage() { - return errorMessage; - } - - - public void setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - } - -} diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java index f538c45d1..808cf45e7 100755 --- a/main/src/com/google/refine/exporters/sql/SqlExporter.java +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -60,6 +60,7 @@ public class SqlExporter implements WriterExporter { private List columnNames = new ArrayList(); private List> sqlDataList = new ArrayList>(); private JSONObject sqlOptions; + @Override public String getContentType() { @@ -72,6 +73,7 @@ public class SqlExporter implements WriterExporter { if(logger.isDebugEnabled()) { logger.debug("export sql with params: {}", params); } + TabularSerializer serializer = new TabularSerializer() { @Override @@ -84,14 +86,14 @@ public class SqlExporter implements WriterExporter { public void endFile() { try { if (columnNames.isEmpty()) { - writer.write(NO_COL_SELECTED_ERROR); logger.error("No Columns Selected!!"); - return; + throw new SqlExporterException(NO_COL_SELECTED_ERROR); + } if (sqlOptions == null) { - writer.write(NO_OPTIONS_PRESENT_ERROR); logger.error("No Options Selected!!"); - return; + throw new SqlExporterException(NO_OPTIONS_PRESENT_ERROR); + } String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName(); @@ -163,11 +165,6 @@ public class SqlExporter implements WriterExporter { CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer); } - - public List validateSqlData(Project project, Properties params){ - - logger.info("Param Name:{}, Param Value:{}", project, params); - return null; - - } + + } diff --git a/main/src/com/google/refine/exporters/sql/SqlExporterException.java b/main/src/com/google/refine/exporters/sql/SqlExporterException.java new file mode 100644 index 000000000..39bdf2cd6 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlExporterException.java @@ -0,0 +1,16 @@ +package com.google.refine.exporters.sql; + + +public class SqlExporterException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = -2640105469265805010L; + + public SqlExporterException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java index efd7686b5..d526a968d 100755 --- a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -53,6 +53,7 @@ public class SqlInsertBuilder { private List> sqlDataList; private JSONObject options; + /** * @@ -60,12 +61,16 @@ public class SqlInsertBuilder { * @param columns * @param rows * @param options + * @param sqlErrors */ - public SqlInsertBuilder(String table, List columns, List> rows, JSONObject options) { + public SqlInsertBuilder(String table, List columns, List> rows, JSONObject options + ) { this.table = table; this.columns = columns; this.sqlDataList = rows; this.options = options; + //logger.info("Column Size:{}", columns.size()); + } /** @@ -86,65 +91,74 @@ public class SqlInsertBuilder { }); } - boolean nullValueToEmptyStr = options == null ? false : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true); + boolean nullValueNull = options == null ? true : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true); StringBuffer values = new StringBuffer(); int idx = 0; for(ArrayList sqlRow : sqlDataList) { StringBuilder rowValue = new StringBuilder(); - + + //int fieldCount = 0; for(SqlData val : sqlRow) { JSONObject jsonOb = colOptionsMap.get(val.getColumnName()); String type = (String)jsonOb.get("type"); - String defaultValue = (String)jsonOb.get("defaultValue"); - boolean allowNullChkBox = (boolean)jsonOb.get("allowNull"); - + String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null); + + boolean allowNullChkBox = JSONUtilities.getBoolean(jsonOb, "defaultValue", true);; if(type == null) { type = SqlData.SQL_TYPE_VARCHAR; } //Character Types if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { - - if(!allowNullChkBox) { - throw new RuntimeException("bad input"); - } - if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) { - // logger.info("Appending empty String:::{}" , val.getText()); - if(defaultValue != null && !defaultValue.isEmpty()) { - rowValue.append("'" + defaultValue + "'"); - }else { - rowValue.append("null"); - } + + if((val.getText() == null || val.getText().isEmpty()) ) { + + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true); }else { rowValue.append("'" + val.getText() + "'"); + } - }else {//Numeric Types + }else if(type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER) || type.equals(SqlData.SQL_TYPE_NUMERIC)) {//Numeric Types : INT, NUMERIC - if((val.getText() == null || val.getText().isEmpty()) && nullValueToEmptyStr ) { - // logger.info("Appending empty String others:::{}" , val.getText()); - if(defaultValue != null && !defaultValue.isEmpty()) { - rowValue.append("'" + defaultValue + "'"); - }else { - rowValue.append("null"); + if((val.getText() == null || val.getText().isEmpty())) { + + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, false); + + }else {//value not null + + try { + Integer.parseInt(val.getText()); + } catch (NumberFormatException nfe) { + throw new SqlExporterException( + val.getText() + " is not compatible with column type :" + type); } - }else { + rowValue.append(val.getText()); + } + }else if(type.equals(SqlData.SQL_TYPE_DATE) || type.equals(SqlData.SQL_TYPE_TIMESTAMP)) { + if((val.getText() == null || val.getText().isEmpty())) { + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true); + }else { + rowValue.append("'" + val.getText() + "'"); + } } rowValue.append(","); - + } + idx++; String rowValString = rowValue.toString(); +// logger.info("rowValString::" + rowValString); rowValString = rowValString.substring(0, rowValString.length() - 1); values.append("( "); @@ -182,4 +196,58 @@ public class SqlInsertBuilder { return sqlString; } + /** + * + * @param allowNullChkBox + * @param defaultValue + * @param nullValueNull + * @param col + * @param rowValue + * @param quote + * @param fieldCount + */ + public void handleNullField( + boolean allowNullChkBox, + String defaultValue, + boolean nullValueNull, + String col, + StringBuilder rowValue, + boolean quote + ) { + + if(allowNullChkBox) {//cell nullable + if(defaultValue != null && !defaultValue.isEmpty()) { + if(quote) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append(defaultValue); + } + + }else { + if(nullValueNull) { + rowValue.append("null"); + + }else { + throw new SqlExporterException("Null value not allowed for Field :" + col); + } + + } + + }else { + if(defaultValue != null && !defaultValue.isEmpty()) { + if(quote) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append(defaultValue); + } + + }else { + throw new SqlExporterException("Null value not allowed for Field :" + col); + } + + } + + } + + } diff --git a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java index e78ebf069..7ddf0e4b5 100644 --- a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java @@ -41,6 +41,8 @@ import java.io.IOException; import java.io.StringWriter; import java.util.List; import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.json.JSONArray; @@ -212,19 +214,23 @@ public class SqlExporterTests extends RefineTest { } String result = writer.toString(); + //logger.info("result = " + result); Assert.assertNotNull(result); - assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); - assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR); +// assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); +// assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR); boolean checkResult = result.contains("CREATE TABLE " + tableName); Assert.assertEquals(checkResult, true); + //logger.info("checkResult1 = " + checkResult); checkResult = result.contains("INSERT INTO " + tableName ); Assert.assertEquals(checkResult, true); + //logger.info("checkResult2 = " + checkResult); - checkResult = result.contains("DROP TABLE " + tableName + ";"); + checkResult = result.contains("DROP TABLE IF EXISTS " + tableName + ";"); Assert.assertEquals(checkResult, true); + //logger.info("checkResult3 = " + checkResult); } @@ -246,8 +252,82 @@ public class SqlExporterTests extends RefineTest { Assert.assertEquals(result, true); } + + @Test + public void testExportSqlWithNullFields(){ + int inNull = 8; + createGridWithNullFields(3, 3, inNull); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + optionsJson.put("convertNulltoEmptyString", true); + + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + Assert.assertNotNull(result); + //logger.info("\nresult = " + result); + // logger.info("\nNull Count:" + countWordInString(result, "null")); + + int countNull = countWordInString(result, "null"); + Assert.assertEquals(countNull, inNull); + + } + + @Test + public void testExportSqlWithNotNullColumns(){ + int noOfCols = 4; + int noOfRows = 3; + createGrid(noOfRows, noOfCols); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + optionsJson.put("convertNulltoEmptyString", true); + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + logger.info("\nresult:={} ", result); + Assert.assertNotNull(result); + + int countNull = countWordInString(result, "NOT NULL"); + logger.info("\nNot Null Count: {}" , countNull); + Assert.assertEquals(countNull, noOfCols); + + } //helper methods + + public int countWordInString(String input, String word){ + if(input == null || input.isEmpty()) { + return 0; + } + int i = 0; + Pattern p = Pattern.compile(word); + Matcher m = p.matcher( input ); + while (m.find()) { + i++; + } + + return i; + + } + protected void createColumns(int noOfColumns){ for(int i = 0; i < noOfColumns; i++){ try { @@ -269,6 +349,27 @@ public class SqlExporterTests extends RefineTest { project.rows.add(row); } } + + protected void createGridWithNullFields(int noOfRows, int noOfColumns, int noOfNullFields){ + createColumns(noOfColumns); + if(noOfNullFields > (noOfColumns * noOfRows)) { + noOfNullFields = noOfColumns * noOfRows; + } + int k = 0; + for(int i = 0; i < noOfRows; i++){ + Row row = new Row(noOfColumns); + for(int j = 0; j < noOfColumns; j++){ + if(k < noOfNullFields) { + row.cells.add(new Cell("", null)); + k++; + }else { + row.cells.add(new Cell("row" + i + "cell" + j, null)); + } + + } + project.rows.add(row); + } + } protected JSONObject createOptionsFromProject(String tableName, String type, String size) { @@ -283,8 +384,17 @@ public class SqlExporterTests extends RefineTest { //logger.info("Column Name = " + c.getName()); JSONObject columnModel = new JSONObject(); columnModel.put("name", c.getName()); - columnModel.put("type", "VARCHAR"); - columnModel.put("size", "100"); + if(type != null) { + columnModel.put("type", type); + }else { + columnModel.put("type", "VARCHAR"); + } + if(size != null) { + columnModel.put("size", size); + }else { + columnModel.put("size", "100"); + } + if(type != null) { columnModel.put("type", type); } @@ -300,5 +410,48 @@ public class SqlExporterTests extends RefineTest { return json; } + protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue, + boolean allowNull) { + + JSONObject json = new JSONObject(); + JSONArray columns = new JSONArray(); + json.put("columns", columns); + json.put("tableName", tableName); + + List cols = project.columnModel.columns; + + cols.forEach(c -> { + //logger.info("Column Name = " + c.getName()); + JSONObject columnModel = new JSONObject(); + columnModel.put("name", c.getName()); + if(type != null) { + columnModel.put("type", type); + }else { + columnModel.put("type", "VARCHAR"); + } + if(size != null) { + columnModel.put("size", size); + }else { + columnModel.put("size", "100"); + } + + if(type != null) { + columnModel.put("type", type); + } + if(size != null) { + // logger.info(" Size = " + size); + columnModel.put("size", size); + } + + columnModel.put("defaultValue", defaultValue); + columnModel.put("allowNull", allowNull); + + columns.put(columnModel); + + }); + + return json; + } + } diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index bffff591f..0f5a9cdae 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -151,8 +151,7 @@ function registerCommands() { RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand()); RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand()); - - RS.registerCommand(module, "preview-sql-export", new Packages.com.google.refine.commands.exporting.SqlExporterCommand()); + } function registerOperations() { diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js index 4e266004c..3bea81a01 100755 --- a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js @@ -336,69 +336,43 @@ function SqlExporterDialog(options) { } var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); - var sqlExportInput = {}; - sqlExportInput.name = name; - sqlExportInput.options = JSON.stringify(options); - sqlExportInput.project = theProject.id; + + var format = options.format; + var encoding = options.encoding; - //validate form @ backend - $.post( - "command/core/preview-sql-export", - sqlExportInput, - function(sqlValidationResult) { - if(sqlValidationResult && sqlValidationResult.code === 'OK'){ - //generate code - - var format = options.format; - var encoding = options.encoding; - - delete options.format; - delete options.encoding; - if (preview) { - options.limit = 10; - } - - // var ext = SqlExporterDialog.formats[format].extension; - var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql"); - $('') - .attr("name", "options") - .attr("value", JSON.stringify(options)) - .appendTo(form); - if (encoding) { - $('') - .attr("name", "encoding") - .attr("value", encoding) - .appendTo(form); - } - if (!preview) { - $('') - .attr("name", "contentType") - .attr("value", "application/x-unknown") // force download - .appendTo(form); - } - - // alert("form::" + form); - document.body.appendChild(form); - - window.open("about:blank", "refine-export"); - form.submit(); - - document.body.removeChild(form); - return true; - - }else{ - window.alert("Problem with sql code genration"); - return false; - } - - }, - "json" - ).fail(function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - return false; - }); + delete options.format; + delete options.encoding; + if (preview) { + options.limit = 10; + } - return true; + // var ext = SqlExporterDialog.formats[format].extension; + var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql"); + $('') + .attr("name", "options") + .attr("value", JSON.stringify(options)) + .appendTo(form); + if (encoding) { + $('') + .attr("name", "encoding") + .attr("value", encoding) + .appendTo(form); + } + if (!preview) { + $('') + .attr("name", "contentType") + .attr("value", "application/x-unknown") // force download + .appendTo(form); + } + + // alert("form::" + form); + document.body.appendChild(form); + + window.open("about:blank", "refine-export"); + form.submit(); + + document.body.removeChild(form); + return true; };