-
Notifications
You must be signed in to change notification settings - Fork 0
/
indexerMulti.py
41 lines (35 loc) · 890 Bytes
/
indexerMulti.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# -*- coding: utf-8 -*-
import pprint
import logging
import urllib
import sys, os
import re
import copy
import json, requests
import MySQLdb
import MySQLdb.cursors
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import elasticsearch
import parse_args
import rules
import loop
import indexer
from es_helper import Es
from multiprocessing import Pool
from multiprocessing import Process
def f(args):
indexer.index(args)
if __name__ == '__main__':
# multiprocessing
args = parse_args.parse()
workerCnt = args['worker_count']
if workerCnt <= 0:
print "worker processes can not be negative!"
elif workerCnt == 1:
args['mod'] = None
Process(target=f, args=(args,)).start()
else:
for num in range(workerCnt):
args['mod'] = [workerCnt, num]
Process(target=f, args=(args,)).start()