s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add UT for issue #1509

Browse Source
This commit is contained in:
Jacky 2018-02-25 15:31:59 -05:00
parent 32c5838a8d
commit 036866837a
4 changed files with 1108 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -1127,6 +1127,19 @@ public class ImportingUtilities {
ProjectManager.singleton.registerProject(project, pm); ProjectManager.singleton.registerProject(project, pm);
// infer the column type // infer the column type
inferColumnType(project);
job.setProjectID(project.id);
job.setState("created-project");
} else {
job.setError(exceptions);
}
job.touch();
job.updating = false;
}
}
public static void inferColumnType(final Project project) {
if (project.columnModel.columns.get(0).getType().isEmpty()) { if (project.columnModel.columns.get(0).getType().isEmpty()) {
List<Object[]> listCells = new ArrayList<Object[]>(INFER_ROW_LIMIT); List<Object[]> listCells = new ArrayList<Object[]>(INFER_ROW_LIMIT);
List<Row> rows = project.rows List<Row> rows = project.rows
@ -1154,15 +1167,6 @@ public class ImportingUtilities {
logger.error("infer column type exception.", ExceptionUtils.getStackTrace(e)); logger.error("infer column type exception.", ExceptionUtils.getStackTrace(e));
} }
} }
job.setProjectID(project.id);
job.setState("created-project");
} else {
job.setError(exceptions);
}
job.touch();
job.updating = false;
}
} }
private static void populateDataPackageMetadata(Project project, ProjectMetadata pmd, DataPackageMetadata metadata) { private static void populateDataPackageMetadata(Project project, ProjectMetadata pmd, DataPackageMetadata metadata) {

1002
main/tests/data/jorf.xml Normal file
View File

File diff suppressed because it is too large Load Diff

7
main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java
View File

@ -5,8 +5,11 @@ import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import static org.mockito.Mockito.when;
import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.mockito.Mockito; import org.mockito.Mockito;
@ -38,7 +41,9 @@ abstract public class ImporterTest extends RefineTest {
ImportingManager.initialize(servlet); ImportingManager.initialize(servlet);
project = new Project(); project = new Project();
metadata = new ProjectMetadata(); metadata = new ProjectMetadata();
job = ImportingManager.createJob(); ImportingJob spiedJob = ImportingManager.createJob();
job = Mockito.spy(spiedJob);
when(job.getRetrievalRecord()).thenReturn(new JSONObject());
options = Mockito.mock(JSONObject.class); options = Mockito.mock(JSONObject.class);
} }

71
main/tests/server/src/com/google/refine/tests/importing/ImportingUtilitiesTests.java
View File

@ -1,21 +1,84 @@
package com.google.refine.tests.importing; package com.google.refine.tests.importing;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.mockito.Mockito;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.importers.XmlImporter;
import com.google.refine.importers.tree.TreeImportingParserBase;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.importing.ImportingUtilities; import com.google.refine.importing.ImportingUtilities;
import com.google.refine.model.Column;
import com.google.refine.model.medadata.ProjectMetadata; import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.tests.RefineTest; import com.google.refine.tests.importers.ImporterTest;
import com.google.refine.util.JSONUtilities;
public class ImportingUtilitiesTests extends RefineTest { public class ImportingUtilitiesTests extends ImporterTest {
@Override
@BeforeMethod
public void setUp(){
super.setUp();
}
@Test @Test
public void createProjectMetadataTest() throws Exception { public void createProjectMetadataTest()
JSONObject optionObj = new JSONObject("{\"projectName\":\"acme\",\"projectTags\":[],\"created\":\"2017-12-18T13:28:40.659\",\"modified\":\"2017-12-20T09:28:06.654\",\"creator\":\"\",\"contributors\":\"\",\"subject\":\"\",\"description\":\"\",\"rowCount\":50,\"customMetadata\":{}}"); throws Exception {
JSONObject optionObj = new JSONObject(
"{\"projectName\":\"acme\",\"projectTags\":[],\"created\":\"2017-12-18T13:28:40.659\",\"modified\":\"2017-12-20T09:28:06.654\",\"creator\":\"\",\"contributors\":\"\",\"subject\":\"\",\"description\":\"\",\"rowCount\":50,\"customMetadata\":{}}");
ProjectMetadata pm = ImportingUtilities.createProjectMetadata(optionObj); ProjectMetadata pm = ImportingUtilities.createProjectMetadata(optionObj);
Assert.assertEquals(pm.getName(), "acme"); Assert.assertEquals(pm.getName(), "acme");
Assert.assertEquals(pm.getEncoding(), "UTF-8"); Assert.assertEquals(pm.getEncoding(), "UTF-8");
Assert.assertTrue(pm.getTags().length == 0); Assert.assertTrue(pm.getTags().length == 0);
} }
@Test
public void inferColumnTypeTest()
throws Exception {
ImportingManager.registerFormat("text/xml", "XML files", "XmlParserUI", new com.google.refine.importers.XmlImporter());
XmlImporter xmlImporter = new XmlImporter();
String fileName = "jorf.xml";
InputStream in = this.getClass().getClassLoader()
.getResourceAsStream(fileName);
options = getNestedOptions(job, xmlImporter);
job.getRetrievalRecord();
parseOneInputStream(new XmlImporter(),
in,
options);
ImportingUtilities.inferColumnType(project);
Assert.assertTrue(project.columnModel.columns.size() == 58);
for (Column c : project.columnModel.columns) {
System.out.println("name:" + c.getName() + ", type:" + c.getType());
}
Assert.assertTrue(project.columnModel.getColumnByName("result - source_id").getType().equals("string"));
Assert.assertTrue(project.columnModel.getColumnByName("result - person - sexe").getType().equals("boolean"));
}
private JSONObject getNestedOptions(ImportingJob job, TreeImportingParserBase parser) {
JSONObject options = parser.createParserUIInitializationData(
job, new LinkedList<JSONObject>(), "text/json");
JSONArray path = new JSONArray();
JSONUtilities.append(path, "results");
JSONUtilities.append(path, "result");
// JSONUtilities.append(path, "object");
JSONUtilities.safePut(options, "recordPath", path);
return options;
}
} }