concordia-server/tests/addSources.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import urllib2
import sys
import time
import host

BUFFER_SIZE = 500


def addSources(sources_buffer):
	data = {
	    'operation': 'addSources',
	    'sources':sources_buffer
	}

	req = urllib2.Request(address)
	req.add_header('Content-Type', 'application/json')
	urllib2.urlopen(req, json.dumps(data))


address = 'http://'+host.concordia_host
if len(host.concordia_port) > 0:
    address += ':'+host.concordia_port


with open(sys.argv[1]) as sources_file:
	counter = 0
	sources_buffer = []
	for line in sources_file:
		counter += 1
		id_raw, link, name = line.rstrip().split('\t')

		sources_buffer.append([int(id_raw),name, link])
		if len(sources_buffer) == BUFFER_SIZE:
			addSources(sources_buffer)
			sources_buffer = []
			print("Added %d sources" % counter)
			

if len(sources_buffer) > 0:
	addSources(sources_buffer)

print("Added all %d sources" % counter)