concordia-docker/tests/addSources.py

50 lines
928 B
Python
Raw Normal View History

2019-05-16 18:28:30 +02:00
#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import urllib2
import sys
import time
import host
BUFFER_SIZE = 500
def addSources(sources_buffer):
data = {
'operation': 'addSources',
'sources':sources_buffer
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data))
address = 'http://'+host.concordia_host
if len(host.concordia_port) > 0:
address += ':'+host.concordia_port
with open(sys.argv[1]) as sources_file:
counter = 0
sources_buffer = []
for line in sources_file:
counter += 1
id_raw, link, name = line.rstrip().split('\t')
sources_buffer.append([int(id_raw),name, link])
if len(sources_buffer) == BUFFER_SIZE:
addSources(sources_buffer)
sources_buffer = []
print("Added %d sources" % counter)
if len(sources_buffer) > 0:
addSources(sources_buffer)
print("Added all %d sources" % counter)