s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

much juicier tests pass and cause the broker to solidify significantly

Browse Source 
yeah, I know, those tests are nasty... but so is what we're trying to do here

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1072 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-07-02 23:24:56 +00:00
parent b0b5e5b781
commit 8997264df0
3 changed files with 191 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java
View File

@ -54,7 +54,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static final public String OPEN = "open"; static final public String OPEN = "open";
static final public int ALL = 0; static final public int ALL = 0;
static final public int COLUMN = 1; static final public int COL = 1;
static final public int CELL = 2; static final public int CELL = 2;
static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker"); static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker");

77
broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java
View File

@ -161,6 +161,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception { protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
logger.trace("> obtain lock"); logger.trace("> obtain lock");
Lock lock = null; Lock lock = null;
Lock blocker = null;
Transaction txn = env.beginTransaction(null, null); Transaction txn = env.beginTransaction(null, null);
@ -168,36 +169,82 @@ public class GridworksBrokerImpl extends GridworksBroker {
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
try { /*
for (Lock l : cursor) { * ALL
logger.info("found lock: {}", l.id); * blocked -> somebody else's lock
* reuse -> you already have an ALL lock
* new -> else
*
* COL
* blocked -> somebody else's all lock || a lock on the same col
* reuse -> you have an ALL lock || a lock on the same col
* new -> else
*
* CELL
* blocked -> somebody else's all lock || a lock on the same col || a lock on the same cell
* reuse -> you have a lock on the same cell
* yes -> (you have a lock on the same cell) && (nobody else has a lock on the same cell || the same col || all)
* new -> else
*
*/
try {
if (locktype == ALL) { if (locktype == ALL) {
for (Lock l : cursor) {
if (!l.uid.equals(uid)) {
blocker = l;
break;
} else {
if (l.type == ALL) { if (l.type == ALL) {
lock = l; lock = l;
break; break;
} }
} else if (locktype == COLUMN) { }
}
} else if (locktype == COL) {
for (Lock l : cursor) {
if (!l.uid.equals(uid)) {
if (l.type == ALL || if (l.type == ALL ||
(l.type == COLUMN && l.value.equals(lockvalue))) { (l.type == COL && l.value.equals(lockvalue))) {
blocker = l;
break;
}
} else {
if (l.type == ALL ||
(l.type == COL && l.value.equals(lockvalue))) {
lock = l; lock = l;
break; break;
} }
}
}
} else if (locktype == CELL) { } else if (locktype == CELL) {
for (Lock l : cursor) {
if (!l.uid.equals(uid)) {
if (l.type == ALL || if (l.type == ALL ||
(l.type == COLUMN && l.value.equals(lockvalue.split(",")[0])) || (l.type == COL && l.value.equals(lockvalue.split(",")[0])) ||
(l.type == CELL && l.value.equals(lockvalue))) { (l.type == CELL && l.value.equals(lockvalue))) {
blocker = l;
break;
}
} else {
if (l.type == CELL && l.value.equals(lockvalue)) {
lock = l; lock = l;
break; break;
} }
} }
} }
}
} finally { } finally {
cursor.close(); cursor.close();
} }
if (blocker != null) {
logger.info("found a blocking lock {}", lockToString(blocker));
throw new RuntimeException("Can't obtain lock, it is blocked by a type '" + blocker.type + "' lock owned by '" + blocker.uid + "'");
}
if (lock == null) { if (lock == null) {
logger.info("no lock found, creating new"); logger.info("no comparable lock already exists, creating a new one");
lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue); lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue);
lockById.put(txn, lock); lockById.put(txn, lock);
txn.commit(); txn.commit();
@ -287,6 +334,8 @@ public class GridworksBrokerImpl extends GridworksBroker {
Lock lock = getLock(lid, pid, uid); Lock lock = getLock(lid, pid, uid);
logger.info("obtained lock: {}", lockToString(lock));
if (lock.type == ALL) { if (lock.type == ALL) {
project.transformations.addAll(transformations); project.transformations.addAll(transformations);
} else { } else {
@ -295,8 +344,8 @@ public class GridworksBrokerImpl extends GridworksBroker {
int type = o.getInt("op_type"); int type = o.getInt("op_type");
String value = o.getString("op_value"); String value = o.getString("op_value");
if (lock.type == COLUMN) { if (lock.type == COL) {
if (type == COLUMN) { if (type == COL) {
if (value != null && value.equals(lock.value)) { if (value != null && value.equals(lock.value)) {
project.transformations.add(s); project.transformations.add(s);
} else { } else {
@ -311,7 +360,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
} }
} }
} else if (lock.type == CELL) { } else if (lock.type == CELL) {
if (type == COLUMN) { if (type == COL) {
throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column."); throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column.");
} else if (type == CELL) { } else if (type == CELL) {
if (value != null && value.equals(lock.value)) { if (value != null && value.equals(lock.value)) {
@ -324,6 +373,8 @@ public class GridworksBrokerImpl extends GridworksBroker {
} }
} }
projectById.put(txn, project);
txn.commit(); txn.commit();
} finally { } finally {
if (txn != null) { if (txn != null) {
@ -374,7 +425,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
writer.array(); writer.array();
int size = project.transformations.size(); int size = project.transformations.size();
for (int i = rev; i < size; i++) { for (int i = rev; i < size; i++) {
writer.value(project.transformations.get(i)); writer.value(new JSONObject(project.transformations.get(i)));
} }
writer.endArray(); writer.endArray();
@ -489,6 +540,10 @@ public class GridworksBrokerImpl extends GridworksBroker {
return o; return o;
} }
String lockToString(Lock lock) {
return lock.id + "," + lock.pid + "," + lock.uid + "," + lock.type + "," + lock.value;
}
@Entity @Entity
static class Lock { static class Lock {

130
broker/core/tests/src/com/metaweb/gridworks/broker/tests/GridworksBrokerTests.java
View File

@ -1,6 +1,6 @@
package com.metaweb.gridworks.broker.tests; package com.metaweb.gridworks.broker.tests;
import static com.metaweb.gridworks.broker.GridworksBroker.ALL; import static com.metaweb.gridworks.broker.GridworksBroker.*;
import static com.metaweb.gridworks.broker.GridworksBroker.EXPIRE; import static com.metaweb.gridworks.broker.GridworksBroker.EXPIRE;
import static com.metaweb.gridworks.broker.GridworksBroker.GET_STATE; import static com.metaweb.gridworks.broker.GridworksBroker.GET_STATE;
import static com.metaweb.gridworks.broker.GridworksBroker.OBTAIN_LOCK; import static com.metaweb.gridworks.broker.GridworksBroker.OBTAIN_LOCK;
@ -109,8 +109,7 @@ public class GridworksBrokerTests {
@Test @Test
public void testService() { public void testService() {
try { try {
JSONObject result = call(broker, request, response, EXPIRE); success(broker, request, response, EXPIRE);
assertJSON(result, "status", "ok");
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -119,8 +118,7 @@ public class GridworksBrokerTests {
@Test @Test
public void testObtainLockFailure() { public void testObtainLockFailure() {
try { try {
JSONObject result = call(broker, request, response, OBTAIN_LOCK); failure(broker, request, response, OBTAIN_LOCK);
assertJSON(result, "status", "error");
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -129,8 +127,7 @@ public class GridworksBrokerTests {
@Test @Test
public void testReleaseLockFailure() { public void testReleaseLockFailure() {
try { try {
JSONObject result = call(broker, request, response, RELEASE_LOCK); failure(broker, request, response, RELEASE_LOCK);
assertJSON(result, "status", "error");
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -139,37 +136,120 @@ public class GridworksBrokerTests {
@Test @Test
public void testStartProject() { public void testStartProject() {
try { try {
String project = "1"; String project = "proj1";
String user = "testuser"; String user = "testuser";
String user2 = "testuser2";
String data = "blah"; String data = "blah";
String metadata = "{}"; String metadata = "{}";
String rev = "0"; String rev = "0";
JSONObject result = call(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", ""); logger.info("--- obtain ALL lock on project ---");
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
assertJSON(result, "uid", "testuser"); assertJSON(result, "uid", "testuser");
String lock = result.getString("lock"); String lock = result.getString("lock");
result = call(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "rev", rev); logger.info("--- start project ---");
assertJSON(result, "status", "ok"); success(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "rev", rev);
result = call(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev); logger.info("--- verify project state contains lock ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
JSONArray locks = result.getJSONArray("locks"); JSONArray locks = result.getJSONArray("locks");
Assert.assertEquals(locks.length(), 1); Assert.assertEquals(locks.length(), 1);
JSONObject l = locks.getJSONObject(0); JSONObject l = locks.getJSONObject(0);
assertJSON(l, "uid", "testuser"); assertJSON(l, "uid", "testuser");
Assert.assertEquals(l.getInt("type"), ALL); Assert.assertEquals(l.getInt("type"), ALL);
result = call(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock); logger.info("--- release ALL lock on project ---");
assertJSON(result, "status", "ok"); success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
result = call(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev); logger.info("--- verify no locks are present ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
locks = result.getJSONArray("locks"); locks = result.getJSONArray("locks");
Assert.assertEquals(locks.length(), 0); Assert.assertEquals(locks.length(), 0);
result = call(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev); logger.info("--- open project and verify data was loaded correctly ---");
assertJSON(result, "status", "ok"); result = success(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev);
JSONArray result_data = result.getJSONArray("data"); JSONArray result_data = result.getJSONArray("data");
Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length); Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length);
JSONArray tt;
JSONObject t;
logger.info("--- obtain column lock ---");
String column = "1";
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", column);
String col_lock = result.getString("lock");
logger.info("--- perform column transformation ---");
t = new JSONObject();
t.put("op_type", COL);
t.put("op_value", column); // operate on col 1
t.put("value", new JSONObject());
tt = new JSONArray();
tt.put(t);
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
logger.info("--- make sure transformation was recorded properly ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
tt = result.getJSONArray("transformations");
Assert.assertEquals(tt.length(), 1);
t = tt.getJSONObject(0);
assertJSON(t, "op_value", column);
logger.info("--- make sure revision numbers in state management work as expected ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
tt = result.getJSONArray("transformations");
Assert.assertEquals(tt.length(), 0);
logger.info("--- perform cell transformation ---");
String cell = "1";
t = new JSONObject();
t.put("op_type", CELL);
t.put("op_value", column + "," + cell); // operate on cell at row 1 column 1
t.put("value", new JSONObject());
tt = new JSONArray();
tt.put(t);
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
logger.info("--- make sure transformation was recorded properly ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
tt = result.getJSONArray("transformations");
Assert.assertEquals(tt.length(), 2);
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
tt = result.getJSONArray("transformations");
Assert.assertEquals(tt.length(), 1);
t = tt.getJSONObject(0);
assertJSON(t, "op_value", column + "," + cell);
logger.info("--- make sure another user fails to acquire ALL lock ---");
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", column);
logger.info("--- make sure another user fails to acquire COL lock on the same column ---");
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column);
logger.info("--- make sure another user manages to acquire COL lock on another column ---");
String column2 = "2";
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column2);
String col_lock2 = result.getString("lock");
logger.info("--- make sure that both locks are present ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
locks = result.getJSONArray("locks");
Assert.assertEquals(locks.length(), 2);
logger.info("--- make sure we can't escalate our current COL lock to an ALL lock ---");
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
logger.info("--- release column locks ---");
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", col_lock);
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", col_lock2);
logger.info("--- make sure the project has no locks ---");
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
locks = result.getJSONArray("locks");
Assert.assertEquals(locks.length(), 0);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -181,7 +261,15 @@ public class GridworksBrokerTests {
Assert.assertEquals(o.get(name), value); Assert.assertEquals(o.get(name), value);
} }
private JSONObject call(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception { private JSONObject success(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
return call(true, broker, request, response, service, params);
}
private JSONObject failure(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
return call(false, broker, request, response, service, params);
}
private JSONObject call(boolean successful, GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
if (params != null) { if (params != null) {
for (int i = 0; i < params.length; ) { for (int i = 0; i < params.length; ) {
String name = params[i++]; String name = params[i++];
@ -206,6 +294,12 @@ public class GridworksBrokerTests {
JSONObject result = new JSONObject(writer.toString()); JSONObject result = new JSONObject(writer.toString());
if (successful) {
assertJSON(result, "status", "ok");
} else {
assertJSON(result, "status", "error");
}
logger.info(result.toString()); logger.info(result.toString());
return result; return result;