s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Store and use job ID to retrieve MDO ID and send that in subsequent loads.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1100 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-07-16 17:32:06 +00:00
parent 4373e7276f
commit 4522b98f32
2 changed files with 48 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
main/src/com/metaweb/gridworks/commands/freebase/UploadDataCommand.java
View File

@ -2,6 +2,7 @@ package com.metaweb.gridworks.commands.freebase;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.Properties;
import javax.servlet.ServletException;
@ -17,9 +18,11 @@ import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.exporters.ProtographTransposeExporter.TripleLoaderExporter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.FreebaseUtils;
import com.metaweb.gridworks.util.ParsingUtilities;
public class UploadDataCommand extends Command {
final static protected String s_dataLoadJobIDPref = "core/freebaseDataLoadJobID";
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -35,17 +38,42 @@ public class UploadDataCommand extends Command {
String source_name = request.getParameter("source_name");
String source_id = request.getParameter("source_id");
String graph = request.getParameter("graph");
String result = FreebaseUtils.uploadTriples(request, graph, source_name, source_id, triples.toString());
String mdo_id = null;
try {
Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(s_dataLoadJobIDPref);
if (jobID != null) {
URL url = new URL("http://gridworks-loads.freebaseapps.com/job_id_to_mdo?job=" + jobID);
String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
if (!s.equals("null")) {
mdo_id = s;
}
}
} catch (Exception e) {
// ignore
}
String uploadResponse = FreebaseUtils.uploadTriples(
request, graph, source_name, source_id, mdo_id, triples.toString());
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
try {
new JSONObject(result);
response.getWriter().write(result);
JSONObject obj = new JSONObject(uploadResponse);
if (obj.has("result") && !obj.isNull("result")) {
JSONObject result = obj.getJSONObject("result");
if (result.has("job_id") && !result.isNull("job_id")) {
Integer jobID = result.getInt("job_id");
project.getMetadata().getPreferenceStore().put(
s_dataLoadJobIDPref, jobID);
}
}
response.getWriter().write(uploadResponse);
} catch (JSONException e) {
respond(response,"500 Error", result);
respond(response,"500 Error", uploadResponse);
}
} catch (Exception e) {

16
main/src/com/metaweb/gridworks/util/FreebaseUtils.java
View File

@ -145,8 +145,14 @@ public class FreebaseUtils {
return EntityUtils.toString(httpResponse.getEntity());
}
public static String uploadTriples(HttpServletRequest request, String graph, String source_name, String source_id, String triples)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
public static String uploadTriples(
HttpServletRequest request,
String graph,
String source_name,
String source_id,
String mdo_id,
String triples
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
@ -168,11 +174,15 @@ public class FreebaseUtils {
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
formparams.add(new BasicNameValuePair("operator", GRIDWORKS_ID));
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
formparams.add(new BasicNameValuePair("software_tool_used", GRIDWORKS_ID));
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("graphport", graph));
formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("check_params", "false"));
if (mdo_id != null) {
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(FREEQ_URL);