AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cleanup.

Browse source 
* Remove useless subscriber
* Fix typo in the config file
* Update Helper accordingly
This commit is contained in:
Raphaël Vinot 2014-08-26 17:36:57 +02:00
parent d7a7da4814
commit 623e876f3b
22 changed files with 10 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
bin/Helper.py
View file

@ -31,6 +31,7 @@ class Redis_Queues(object):
self.subscriber_name = subscriber_name
self.sub_channel = self.config.get(conf_section, conf_channel)
self.sub_address = self.config.get(conf_section, 'address')
self.redis_channel = self.sub_channel + self.subscriber_name
# Redis Queue
@ -40,11 +41,10 @@ class Redis_Queues(object):
port=self.config.getint(config_section, "port"),
db=self.config.getint(config_section, "db"))
def zmq_sub(self, conf_section):
sub_address = self.config.get(conf_section, 'adress')
def zmq_sub(self):
context = zmq.Context()
self.sub_socket = context.socket(zmq.SUB)
self.sub_socket.connect(sub_address)
self.sub_socket.connect(self.sub_address)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.sub_channel)
def zmq_pub(self, config_section, config_channel):
@ -74,6 +74,7 @@ class Redis_Queues(object):
return self.r_queues.srem('SHUTDOWN_FLAGS', flag)
def redis_queue_subscribe(self, publisher):
self.zmq_sub()
publisher.info("Suscribed to channel {}".format(self.sub_channel))
while True:
msg = self.sub_socket.recv()

1
bin/ZMQ_Feed_Q.py
View file

@ -34,5 +34,4 @@ if __name__ == "__main__":
subscriber_name = 'feed'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

1
bin/ZMQ_PubSub_Categ_Q.py
View file

@ -32,5 +32,4 @@ if __name__ == "__main__":
subscriber_name = 'categ'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_PubSub_Lines.py
View file

@ -48,9 +48,6 @@ if __name__ == "__main__":
pub_config_section = 'PubSub_Longlines'
h.zmq_pub(pub_config_section, None)
# Subscriber
h.zmq_sub(config_section)
# SCRIPT PARSER #
parser = argparse.ArgumentParser(
description='''This script is a part of the Analysis Information \

1
bin/ZMQ_PubSub_Lines_Q.py
View file

@ -31,5 +31,4 @@ if __name__ == "__main__":
subscriber_name = 'line'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

1
bin/ZMQ_PubSub_Tokenize_Q.py
View file

@ -32,5 +32,4 @@ if __name__ == "__main__":
subscriber_name = 'tokenize'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Attributes.py
View file

@ -42,9 +42,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# FUNCTIONS #
publisher.info("""ZMQ Attribute is Running""")

1
bin/ZMQ_Sub_Attributes_Q.py
View file

@ -32,5 +32,4 @@ if __name__ == "__main__":
subscriber_name = 'attributes'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_CreditCards.py
View file

@ -18,9 +18,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# FUNCTIONS #
publisher.info("Creditcard script subscribed to channel creditcard_categ")

1
bin/ZMQ_Sub_CreditCards_Q.py
View file

@ -15,5 +15,4 @@ if __name__ == "__main__":
subscriber_name = 'cards'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Curve.py
View file

@ -41,9 +41,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# REDIS #
r_serv1 = redis.StrictRedis(
host=h.config.get("Redis_Level_DB", "host"),

1
bin/ZMQ_Sub_Curve_Q.py
View file

@ -32,5 +32,4 @@ if __name__ == "__main__":
subscriber_name = 'curve'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Duplicate.py
View file

@ -31,9 +31,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# REDIS #
# DB OBJECT & HASHS ( DISK )
# FIXME increase flexibility

1
bin/ZMQ_Sub_Duplicate_Q.py
View file

@ -14,5 +14,4 @@ if __name__ == "__main__":
subscriber_name = 'duplicate'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

1
bin/ZMQ_Sub_Indexer_Q.py
View file

@ -26,5 +26,4 @@ if __name__ == "__main__":
subscriber_name = 'indexer'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Mails.py
View file

@ -21,9 +21,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# REDIS #
r_serv2 = redis.StrictRedis(
host=h.config.get("Redis_Cache", "host"),

1
bin/ZMQ_Sub_Mails_Q.py
View file

@ -14,5 +14,4 @@ if __name__ == "__main__":
subscriber_name = 'emails'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Onion.py
View file

@ -39,9 +39,6 @@ if __name__ == "__main__":
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
# Subscriber
h.zmq_sub(config_section)
# FUNCTIONS #
publisher.info("Script subscribed to channel onion_categ")

1
bin/ZMQ_Sub_Onion_Q.py
View file

@ -31,5 +31,4 @@ if __name__ == "__main__":
subscriber_name = 'tor'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

3
bin/ZMQ_Sub_Urls.py
View file

@ -31,9 +31,6 @@ if __name__ == "__main__":
pub_config_channel = 'channel'
h.zmq_pub(pub_config_section, pub_config_channel)
# Subscriber
h.zmq_sub(config_section)
# REDIS #
r_serv2 = redis.StrictRedis(
host=h.config.get("Redis_Cache", "host"),

1
bin/ZMQ_Sub_Urls_Q.py
View file

@ -15,5 +15,4 @@ if __name__ == "__main__":
subscriber_name = 'urls'
h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
h.zmq_sub(config_section)
h.redis_queue_subscribe(publisher)

12
bin/packages/config.cfg.sample
View file

@ -38,31 +38,31 @@ db = 1
# PUB / SUB : ZMQ
[Feed]
adress = tcp://crf.circl.lu:5556
address = tcp://crf.circl.lu:5556
topicfilter = 102
[PubSub_Global]
adress = tcp://127.0.0.1:5000
address = tcp://127.0.0.1:5000
channel = filelist
[PubSub_Longlines]
adress = tcp://127.0.0.1:5001
address = tcp://127.0.0.1:5001
channel_0 = Longlines
channel_1 = Shortlines
[PubSub_Words]
adress = tcp://127.0.0.1:5002
address = tcp://127.0.0.1:5002
channel_0 = words
[PubSub_Categ]
adress = tcp://127.0.0.1:5003
address = tcp://127.0.0.1:5003
channel_0 = creditcard_categ
channel_1 = mails_categ
channel_2 = onion_categ
channel_3 = web_categ
[PubSub_Url]
adress = tcp://127.0.0.1:5004
address = tcp://127.0.0.1:5004
channel = urls
# country code logged as critical
cc_critical = DE