s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1234 from jackyq2015/master

Browse Source 
fix issue #137
This commit is contained in:
Jacky 2017-08-13 14:10:27 -04:00 committed by GitHub
commit 65e22ae5b6
5 changed files with 46 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.classpath
View File

@ -84,5 +84,6 @@
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/> <classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.13-20150929.jar"/> <classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.13-20150929.jar"/>
<classpathentry kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.7.0.jar"/> <classpathentry kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.7.0.jar"/>
<classpathentry kind="lib" path="main/tests/data"/>
<classpathentry kind="output" path="build"/> <classpathentry kind="output" path="build"/>
</classpath> </classpath>

1
build.xml
View File

@ -93,6 +93,7 @@
<path refid="server.class.path"/> <path refid="server.class.path"/>
<pathelement location="${server.tests.classes.dir}"/> <pathelement location="${server.tests.classes.dir}"/>
<pathelement location="${server.tests.lib.dir}/testng-6.8.jar"/> <pathelement location="${server.tests.lib.dir}/testng-6.8.jar"/>
<pathelement location="${tests.dir}/data" />
<fileset dir="${server.tests.lib.dir}"> <fileset dir="${server.tests.lib.dir}">
<include name="**/*.jar" /> <include name="**/*.jar" />
</fileset> </fileset>

3
main/src/com/google/refine/model/RecordModel.java
View File

@ -155,7 +155,8 @@ public class RecordModel implements Jsonizable {
for (int g = 0; g < groupCount; g++) { for (int g = 0; g < groupCount; g++) {
KeyedGroup group = keyedGroups.get(g); KeyedGroup group = keyedGroups.get(g);
if (!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
int contextRowIndex = lastNonBlankRowsByGroup[g]; int contextRowIndex = lastNonBlankRowsByGroup[g];
if (contextRowIndex >= 0) { if (contextRowIndex >= 0) {
for (int dependentCellIndex : group.cellIndices) { for (int dependentCellIndex : group.cellIndices) {

11
main/tests/data/grid_small.json Normal file
View File

@ -0,0 +1,11 @@
{"version": "release_2017_07_12",
"institutes": [
{"name":"Australian National University","wikipedia_url":"http://en.wikipedia.org/wiki/Australian_National_University","email_address":null,"links":["http://www.anu.edu.au/"],"aliases":[],"acronyms":["ANU"],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-35.2778,"lng":149.1205,"postcode":"","primary":false,"city":"Canberra","state":"Australian Capital Territory","state_code":"AU-ACT","country":"Australia","country_code":"AU","geonames_city":{"id":2172517,"city":"Canberra","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"Australian Capital Territory","ascii_name":"Australian Capital Territory","code":"AU.01"},"geonames_admin2":null,"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1001.0","status":"active","established":1946,"relationships":[{"type":"Related","label":"Calvary Hospital","id":"grid.460694.9"},{"type":"Related","label":"Canberra Hospital","id":"grid.413314.0"},{"type":"Related","label":"Goulburn Base Hospital","id":"grid.460721.6"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0001 2180 7477"]},"FundRef":{"preferred":"501100000995","all":["501100000995","100009020","501100001151"]},"OrgRef":{"preferred":null,"all":["285106"]},"Wikidata":{"preferred":null,"all":["Q127990"]}},"weight":1},
{"name":"Monash University","wikipedia_url":"http://en.wikipedia.org/wiki/Monash_University","email_address":null,"links":["http://www.monash.edu/"],"aliases":[],"acronyms":[],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-37.9083,"lng":145.138,"postcode":"","primary":false,"city":"Melbourne","state":"Victoria","state_code":"AU-VIC","country":"Australia","country_code":"AU","geonames_city":{"id":2158177,"city":"Melbourne","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"Victoria","ascii_name":"Victoria","code":"AU.07"},"geonames_admin2":{"name":"Melbourne","ascii_name":"Melbourne","code":"AU.07.24600"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1002.3","status":"active","established":1958,"relationships":[{"type":"Related","label":"Alfred Hospital","id":"grid.1623.6"},{"type":"Related","label":"Box Hill Hospital","id":"grid.414580.c"},{"type":"Related","label":"Burnet Institute","id":"grid.1056.2"},{"type":"Related","label":"Frankston Hospital","id":"grid.415031.2"},{"type":"Related","label":"Jessie McPherson Private Hospital","id":"grid.460735.3"},{"type":"Related","label":"Monash Institute of Medical Research","id":"grid.416057.1"},{"type":"Related","label":"Monash Medical Centre","id":"grid.416060.5"},{"type":"Related","label":"Monash South Africa","id":"grid.470947.9"},{"type":"Child","label":"IITB-Monash Research Academy","id":"grid.464914.a"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0004 1936 7857"]},"FundRef":{"preferred":"501100001779","all":["501100001779","501100001144","501100007917","501100006532","501100000991","501100001198"]},"OrgRef":{"preferred":null,"all":["149620"]},"Wikidata":{"preferred":null,"all":["Q598841"]}},"weight":1},
{"name":"University of Queensland","wikipedia_url":"http://en.wikipedia.org/wiki/University_of_Queensland","email_address":"","links":["http://www.uq.edu.au/"],"aliases":[],"acronyms":["UQ"],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-27.495964,"lng":153.009627,"postcode":"","primary":false,"city":"Brisbane","state":"Queensland","state_code":"AU-QLD","country":"Australia","country_code":"AU","geonames_city":{"id":2174003,"city":"Brisbane","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"Queensland","ascii_name":"Queensland","code":"AU.04"},"geonames_admin2":{"name":"Brisbane","ascii_name":"Brisbane","code":"AU.04.31000"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1003.2","status":"active","established":1909,"relationships":[{"type":"Related","label":"Ipswich Hospital","id":"grid.460731.7"},{"type":"Related","label":"Ochsner Medical Center","id":"grid.240416.5"},{"type":"Related","label":"Princess Alexandra Hospital","id":"grid.412744.0"},{"type":"Related","label":"Royal Brisbane and Women's Hospital","id":"grid.416100.2"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0000 9320 7537"]},"FundRef":{"preferred":"501100001794","all":["501100001794","501100008407","501100005268"]},"OrgRef":{"preferred":null,"all":["192819"]},"Wikidata":{"preferred":null,"all":["Q866012"]}},"weight":1},
{"name":"Macquarie University","wikipedia_url":"http://en.wikipedia.org/wiki/Macquarie_University","email_address":null,"links":["http://mq.edu.au/"],"aliases":[],"acronyms":[],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-33.775259,"lng":151.112915,"postcode":"","primary":false,"city":"Sydney","state":"New South Wales","state_code":"AU-NSW","country":"Australia","country_code":"AU","geonames_city":{"id":2147714,"city":"Sydney","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"New South Wales","ascii_name":"New South Wales","code":"AU.02"},"geonames_admin2":{"name":"City of Sydney","ascii_name":"City of Sydney","code":"AU.02.17200"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1004.5","status":"active","established":1964,"relationships":[{"type":"Related","label":"Sydney Hospital","id":"grid.416790.d"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0001 2158 5405"]},"FundRef":{"preferred":null,"all":["501100001230"]},"OrgRef":{"preferred":null,"all":["19735"]},"Wikidata":{"preferred":null,"all":["Q741082"]}},"weight":1},
{"name":"UNSW Australia","wikipedia_url":"http://en.wikipedia.org/wiki/University_of_New_South_Wales","email_address":null,"links":["https://www.unsw.edu.au/"],"aliases":["University of New South Wales"],"acronyms":["UNSW"],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-33.917731,"lng":151.230964,"postcode":"","primary":false,"city":"Sydney","state":"New South Wales","state_code":"AU-NSW","country":"Australia","country_code":"AU","geonames_city":{"id":2147714,"city":"Sydney","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"New South Wales","ascii_name":"New South Wales","code":"AU.02"},"geonames_admin2":{"name":"City of Sydney","ascii_name":"City of Sydney","code":"AU.02.17200"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1005.4","status":"active","established":1949,"relationships":[{"type":"Related","label":"Bankstown Lidcombe Hospital","id":"grid.414201.2"},{"type":"Related","label":"Black Dog Institute","id":"grid.418393.4"},{"type":"Related","label":"Children's Cancer Institute Australia","id":"grid.413950.a"},{"type":"Related","label":"Fairfield Hospital","id":"grid.432149.9"},{"type":"Related","label":"Ingham Institute","id":"grid.429098.e"},{"type":"Related","label":"Kenvale College","id":"grid.460744.5"},{"type":"Related","label":"Neuroscience Research Australia","id":"grid.250407.4"},{"type":"Related","label":"Prince of Wales Hospital","id":"grid.415193.b"},{"type":"Related","label":"Royal Hospital for Women","id":"grid.416139.8"},{"type":"Related","label":"Shellharbour Hospital","id":"grid.460804.e"},{"type":"Related","label":"Shoalhaven District Memorial Hospital","id":"grid.460806.c"},{"type":"Related","label":"St George Hospital","id":"grid.416398.1"},{"type":"Related","label":"Sutherland Hospital","id":"grid.460648.8"},{"type":"Related","label":"Sydney Children's Hospital","id":"grid.414009.8"},{"type":"Related","label":"Sydney Hospital","id":"grid.416790.d"},{"type":"Related","label":"Victor Chang Cardiac Research Institute","id":"grid.1057.3"},{"type":"Related","label":"Wagga Wagga Rural Referral Hospital","id":"grid.460669.b"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0004 4902 0432"]},"FundRef":{"preferred":"501100001773","all":["501100001773","501100001143","501100007281","501100002204","501100001156"]},"OrgRef":{"preferred":"146078","all":["146078","16766976"]},"Wikidata":{"preferred":null,"all":["Q734764"]}},"weight":1},
{"name":"Newcastle University","wikipedia_url":"http://en.wikipedia.org/wiki/Newcastle_University","email_address":null,"links":["http://www.ncl.ac.uk/"],"aliases":["University of Newcastle upon Tyne"],"acronyms":[],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":54.978,"lng":-1.615,"postcode":"","primary":false,"city":"Newcastle upon Tyne","state":null,"state_code":"","country":"United Kingdom","country_code":"GB","geonames_city":{"id":2641673,"city":"Newcastle upon Tyne","nuts_level1":{"name":"North East (England)","ascii_name":"North East (England)","code":"UKC"},"nuts_level2":{"name":"Northumberland and Tyne and Wear","ascii_name":"Northumberland and Tyne and Wear","code":"UKC2"},"nuts_level3":{"name":"Tyneside","ascii_name":"Tyneside","code":"UKC22"},"geonames_admin1":{"name":"England","ascii_name":"England","code":"GB.ENG"},"geonames_admin2":{"name":"Newcastle upon Tyne","ascii_name":"Newcastle upon Tyne","code":"GB.ENG.I7"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1006.7","status":"active","established":1834,"relationships":[{"type":"Related","label":"Freeman Hospital","id":"grid.415050.5"},{"type":"Related","label":"NIHR Newcastle Dementia Biomedical Research Unit","id":"grid.454379.8"},{"type":"Related","label":"Newcastle Hospitals - Campus for Ageing and Vitality","id":"grid.156122.3"},{"type":"Related","label":"Newcastle University Medicine Malaysia","id":"grid.472342.4"},{"type":"Related","label":"Newcastle University Singapore","id":"grid.473733.7"},{"type":"Related","label":"Queen Elizabeth Hospital","id":"grid.415506.3"},{"type":"Related","label":"Royal Victoria Infirmary","id":"grid.419334.8"},{"type":"Related","label":"University Hospital of North Durham","id":"grid.414158.d"},{"type":"Related","label":"Wellcome Trust Centre for Mitochondrial Research","id":"grid.450004.5"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0001 0462 7212"]},"FundRef":{"preferred":null,"all":["501100008406","501100000774"]},"HESA":{"preferred":null,"all":["0154"]},"UCAS":{"preferred":null,"all":["N21"]},"UKPRN":{"preferred":null,"all":["10007799"]},"OrgRef":{"preferred":"203338","all":["203338","4329975"]},"Wikidata":{"preferred":null,"all":["Q837164"]}},"weight":1},
{"name":"University of Wollongong","wikipedia_url":"http://en.wikipedia.org/wiki/University_of_Wollongong","email_address":null,"links":["https://www.uow.edu.au/index.html"],"aliases":["Wollongong University"],"acronyms":["UOW"],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-34.405147,"lng":150.878387,"postcode":"","primary":false,"city":"Wollongong","state":"New South Wales","state_code":"AU-NSW","country":"Australia","country_code":"AU","geonames_city":{"id":2171507,"city":"Wollongong","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"New South Wales","ascii_name":"New South Wales","code":"AU.02"},"geonames_admin2":{"name":"Wollongong","ascii_name":"Wollongong","code":"AU.02.18450"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1007.6","status":"active","established":1951,"relationships":[{"type":"Related","label":"Shellharbour Hospital","id":"grid.460804.e"},{"type":"Related","label":"Shoalhaven District Memorial Hospital","id":"grid.460806.c"},{"type":"Related","label":"Wollongong Hospital","id":"grid.417154.2"},{"type":"Child","label":"University of Wollongong in Dubai","id":"grid.444532.0"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0004 0486 528X"]},"FundRef":{"preferred":null,"all":["501100001777"]},"OrgRef":{"preferred":null,"all":["461935"]},"Wikidata":{"preferred":null,"all":["Q1350021"]}},"weight":1},
{"name":"University of Melbourne","wikipedia_url":"http://en.wikipedia.org/wiki/University_of_Melbourne","email_address":"","links":["http://www.unimelb.edu.au/"],"aliases":["Melbourne University"],"acronyms":[],"types":["Education"],"ip_addresses":[],"addresses":[{"line_1":"","line_2":"","line_3":null,"lat":-37.797115,"lng":144.959972,"postcode":"","primary":false,"city":"Melbourne","state":"Victoria","state_code":"AU-VIC","country":"Australia","country_code":"AU","geonames_city":{"id":2158177,"city":"Melbourne","nuts_level1":null,"nuts_level2":null,"nuts_level3":null,"geonames_admin1":{"name":"Victoria","ascii_name":"Victoria","code":"AU.07"},"geonames_admin2":{"name":"Melbourne","ascii_name":"Melbourne","code":"AU.07.24600"},"license":{"attribution":"Data from geonames.org under a CC-BY 3.0 license","license":"http://creativecommons.org/licenses/by/3.0/"}}}],"labels":[],"id":"grid.1008.9","status":"active","established":1853,"relationships":[{"type":"Related","label":"Austin Health","id":"grid.410678.c"},{"type":"Related","label":"Austin Hospital","id":"grid.414094.c"},{"type":"Related","label":"Bionics Institute","id":"grid.431365.6"},{"type":"Related","label":"Burnet Institute","id":"grid.1056.2"},{"type":"Related","label":"Centre for Eye Research Australia","id":"grid.418002.f"},{"type":"Related","label":"Epworth Hospital","id":"grid.414539.e"},{"type":"Related","label":"Geelong Hospital","id":"grid.415335.5"},{"type":"Related","label":"Melbourne Health","id":"grid.429299.d"},{"type":"Related","label":"Mental Health Research Institute","id":"grid.415325.4"},{"type":"Related","label":"Northern Hospital","id":"grid.416536.3"},{"type":"Related","label":"Royal Children's Hospital","id":"grid.416107.5"},{"type":"Related","label":"Royal Women's Hospital","id":"grid.416259.d"},{"type":"Related","label":"St Vincent's Hospital","id":"grid.413105.2"},{"type":"Related","label":"St Vincents Institute of Medical Research","id":"grid.1073.5"},{"type":"Related","label":"The Royal Victorian Eye \u0026 Ear Hospital","id":"grid.410670.4"},{"type":"Child","label":"Peter Doherty Institute","id":"grid.483778.7"}],"external_ids":{"ISNI":{"preferred":null,"all":["0000 0001 2179 088X"]},"FundRef":{"preferred":"501100001782","all":["501100001782","501100007961","100008758","501100000986","501100000987","501100001052"]},"OrgRef":{"preferred":"363594","all":["363594","47000931","3917352","1007776","6839809","6954088"]},"Wikidata":{"preferred":null,"all":["Q319078"]}},"weight":1}
]}

34
main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java
View File

@ -35,6 +35,7 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.LinkedList; import java.util.LinkedList;
@ -363,6 +364,21 @@ public class JsonImporterTests extends ImporterTest {
// TODO: check data types // TODO: check data types
} }
@Test
public void testComplexJsonStructure() throws IOException{
String fileName = "grid_small.json";
RunComplexJSONTest(getComplexJSON(fileName));
log(project);
logger.info("************************ columnu number:" + project.columnModel.columns.size() +
". \tcolumn groups number:" + project.columnModel.columnGroups.size() +
".\trow number:" + project.rows.size() + ".\trecord number:" + project.recordModel.getRecordCount()) ;
assertProjectCreated(project, 63, 63, 8);
}
//------------helper methods--------------- //------------helper methods---------------
private static String getTypicalElement(int id){ private static String getTypicalElement(int id){
@ -397,13 +413,13 @@ public class JsonImporterTests extends ImporterTest {
return sb.toString(); return sb.toString();
} }
private static JSONObject getOptions(ImportingJob job, TreeImportingParserBase parser) { private static JSONObject getOptions(ImportingJob job, TreeImportingParserBase parser, String pathSelector) {
JSONObject options = parser.createParserUIInitializationData( JSONObject options = parser.createParserUIInitializationData(
job, new LinkedList<JSONObject>(), "text/json"); job, new LinkedList<JSONObject>(), "text/json");
JSONArray path = new JSONArray(); JSONArray path = new JSONArray();
JSONUtilities.append(path, JsonImporter.ANONYMOUS); JSONUtilities.append(path, JsonImporter.ANONYMOUS);
JSONUtilities.append(path, JsonImporter.ANONYMOUS); JSONUtilities.append(path, pathSelector);
JSONUtilities.safePut(options, "recordPath", path); JSONUtilities.safePut(options, "recordPath", path);
JSONUtilities.safePut(options, "trimStrings", false); JSONUtilities.safePut(options, "trimStrings", false);
@ -493,7 +509,11 @@ public class JsonImporterTests extends ImporterTest {
private void RunTest(String testString) { private void RunTest(String testString) {
RunTest(testString, getOptions(job, SUT)); RunTest(testString, getOptions(job, SUT, JsonImporter.ANONYMOUS));
}
private void RunComplexJSONTest(String testString) {
RunTest(testString, getOptions(job, SUT, "institutes"));
} }
private void RunTest(String testString, JSONObject options) { private void RunTest(String testString, JSONObject options) {
@ -509,4 +529,12 @@ public class JsonImporterTests extends ImporterTest {
Assert.fail(); Assert.fail();
} }
} }
private String getComplexJSON(String fileName) throws IOException {
InputStream in = this.getClass().getClassLoader()
.getResourceAsStream(fileName);
String content = org.apache.commons.io.IOUtils.toString(in);
return content;
}
} }