50 lines
928 B
Python
50 lines
928 B
Python
|
#!/usr/bin/python
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
import json
|
||
|
|
import urllib2
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
import host
|
||
|
|
|
||
|
|
BUFFER_SIZE = 500
|
||
|
|
|
||
|
|
|
||
|
|
def addSources(sources_buffer):
|
||
|
|
data = {
|
||
|
|
'operation': 'addSources',
|
||
|
|
'sources':sources_buffer
|
||
|
|
}
|
||
|
|
|
||
|
|
req = urllib2.Request(address)
|
||
|
|
req.add_header('Content-Type', 'application/json')
|
||
|
|
urllib2.urlopen(req, json.dumps(data))
|
||
|
|
|
||
|
|
|
||
|
|
address = 'http://'+host.concordia_host
|
||
|
|
if len(host.concordia_port) > 0:
|
||
|
|
address += ':'+host.concordia_port
|
||
|
|
|
||
|
|
|
||
|
|
with open(sys.argv[1]) as sources_file:
|
||
|
|
counter = 0
|
||
|
|
sources_buffer = []
|
||
|
|
for line in sources_file:
|
||
|
|
counter += 1
|
||
|
|
id_raw, link, name = line.rstrip().split('\t')
|
||
|
|
|
||
|
|
sources_buffer.append([int(id_raw),name, link])
|
||
|
|
if len(sources_buffer) == BUFFER_SIZE:
|
||
|
|
addSources(sources_buffer)
|
||
|
|
sources_buffer = []
|
||
|
|
print("Added %d sources" % counter)
|
||
|
|
|
||
|
|
|
||
|
|
if len(sources_buffer) > 0:
|
||
|
|
addSources(sources_buffer)
|
||
|
|
|
||
|
|
print("Added all %d sources" % counter)
|
||
|
|
|
||
|
|
|
||
|
|
|