concordia-docker/tools/addSources.py
Rafał Jaworski 6b59e1b0f3 cleaned tests
2019-05-30 11:25:12 +02:00

50 lines
928 B
Python
Executable File

#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import urllib2
import sys
import time
import host
BUFFER_SIZE = 500
def addSources(sources_buffer):
data = {
'operation': 'addSources',
'sources':sources_buffer
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data))
address = 'http://'+host.concordia_host
if len(host.concordia_port) > 0:
address += ':'+host.concordia_port
with open(sys.argv[1]) as sources_file:
counter = 0
sources_buffer = []
for line in sources_file:
counter += 1
id_raw, link, name = line.rstrip().split('\t')
sources_buffer.append([int(id_raw),name, link])
if len(sources_buffer) == BUFFER_SIZE:
addSources(sources_buffer)
sources_buffer = []
print("Added %d sources" % counter)
if len(sources_buffer) > 0:
addSources(sources_buffer)
print("Added all %d sources" % counter)