From 3886d1b834c1c8239b37935890b507c1aa844156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Aug 2014 17:35:40 +0200 Subject: [PATCH] Small fixes to make the refactoring production ready * the port for the logging is 6380 * use os.environ properly * fix typos --- bin/Dir.py | 1 + bin/Helper.py | 3 ++- bin/Queues_Monitoring.py | 1 + bin/Repartition_graph.py | 1 + bin/Shutdown.py | 2 +- bin/ZMQ_Feed.py | 3 ++- bin/ZMQ_Feed_Q.py | 2 +- bin/ZMQ_PubSub_Categ.py | 6 +++--- bin/ZMQ_PubSub_Categ_Q.py | 1 + bin/ZMQ_PubSub_Lines.py | 1 + bin/ZMQ_PubSub_Lines_Q.py | 1 + bin/ZMQ_PubSub_Tokenize.py | 1 + bin/ZMQ_PubSub_Tokenize_Q.py | 1 + bin/ZMQ_Pub_Global.py | 1 + bin/ZMQ_Sub_Attributes.py | 1 + bin/ZMQ_Sub_Attributes_Q.py | 1 + bin/ZMQ_Sub_CreditCards.py | 1 + bin/ZMQ_Sub_CreditCards_Q.py | 1 + bin/ZMQ_Sub_Curve.py | 5 +++-- bin/ZMQ_Sub_Curve_Q.py | 1 + bin/ZMQ_Sub_Duplicate.py | 3 ++- bin/ZMQ_Sub_Duplicate_Q.py | 1 + bin/ZMQ_Sub_Indexer.py | 1 + bin/ZMQ_Sub_Indexer_Q.py | 1 + bin/ZMQ_Sub_Mails.py | 1 + bin/ZMQ_Sub_Mails_Q.py | 1 + bin/ZMQ_Sub_Onion.py | 1 + bin/ZMQ_Sub_Onion_Q.py | 1 + bin/ZMQ_Sub_Urls.py | 1 + bin/ZMQ_Sub_Urls_Q.py | 1 + bin/indexer_lookup.py | 4 ++-- bin/packages/Paste.py | 2 +- 32 files changed, 40 insertions(+), 13 deletions(-) diff --git a/bin/Dir.py b/bin/Dir.py index 37354f6e..6156c579 100755 --- a/bin/Dir.py +++ b/bin/Dir.py @@ -38,6 +38,7 @@ def main(): port=cfg.getint("Redis_Queues", "port"), db=cfg.getint("Redis_Queues", "db")) + publisher.port = 6380 publisher.channel = "Script" create_dirfile(r_serv, args.directory, args.ow) diff --git a/bin/Helper.py b/bin/Helper.py index d04b8675..92ce56c1 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -20,7 +20,8 @@ import zmq class Redis_Queues(object): def __init__(self, conf_section, conf_channel, subscriber_name): - configfile = os.path.join(os.environ('AIL_BIN'), 'packages/config.cfg') + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + print configfile if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ diff --git a/bin/Queues_Monitoring.py b/bin/Queues_Monitoring.py index 2439e838..955f3ed5 100755 --- a/bin/Queues_Monitoring.py +++ b/bin/Queues_Monitoring.py @@ -33,6 +33,7 @@ def main(): db=cfg.getint("Redis_Queues", "db")) # LOGGING # + publisher.port = 6380 publisher.channel = "Queuing" # ZMQ # diff --git a/bin/Repartition_graph.py b/bin/Repartition_graph.py index 89a661c7..f1955a0e 100755 --- a/bin/Repartition_graph.py +++ b/bin/Repartition_graph.py @@ -34,6 +34,7 @@ def main(): db=cfg.getint("Redis_Level_DB_Hashs", "db")) # LOGGING # + publisher.port = 6380 publisher.channel = "Graph" # FUNCTIONS # diff --git a/bin/Shutdown.py b/bin/Shutdown.py index 108801da..8467dafb 100755 --- a/bin/Shutdown.py +++ b/bin/Shutdown.py @@ -24,7 +24,7 @@ import redis import ConfigParser import os -configfile = os.path.join(os.environ('AIL_BIN'), './packages/config.cfg') +configfile = os.path.join(os.environ['AIL_BIN'], './packages/config.cfg') def main(): diff --git a/bin/ZMQ_Feed.py b/bin/ZMQ_Feed.py index 1587bb45..aac05045 100755 --- a/bin/ZMQ_Feed.py +++ b/bin/ZMQ_Feed.py @@ -29,6 +29,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'Feed' @@ -67,7 +68,7 @@ if __name__ == "__main__": time.sleep(10) continue # Creating the full filepath - filename = os.path.join(os.environ('AIL_HOME'), + filename = os.path.join(os.environ['AIL_HOME'], h.config.get("Directories", "pastes"), paste) dirname = os.path.dirname(filename) if not os.path.exists(dirname): diff --git a/bin/ZMQ_Feed_Q.py b/bin/ZMQ_Feed_Q.py index ba7c4a2d..9767c059 100755 --- a/bin/ZMQ_Feed_Q.py +++ b/bin/ZMQ_Feed_Q.py @@ -33,6 +33,6 @@ if __name__ == "__main__": config_channel = 'topicfilter' subscriber_name = 'feed' - h = Helper.Redis_Queues(subscriber_name) + h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_PubSub_Categ.py b/bin/ZMQ_PubSub_Categ.py index 8a8d91c1..71c4c164 100755 --- a/bin/ZMQ_PubSub_Categ.py +++ b/bin/ZMQ_PubSub_Categ.py @@ -36,7 +36,6 @@ Requirements *Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly. """ -import glob import os import argparse import time @@ -46,6 +45,7 @@ from packages import Paste import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Words' @@ -75,10 +75,10 @@ if __name__ == "__main__": "Script Categ subscribed to channel {}".format(h.sub_channel)) tmp_dict = {} - for filename in glob.glob(args.d): + for filename in os.listdir(args.d): bname = os.path.basename(filename) tmp_dict[bname] = [] - with open(filename, 'r') as f: + with open(os.path.join(args.d, filename), 'r') as f: for l in f: tmp_dict[bname].append(l.strip()) diff --git a/bin/ZMQ_PubSub_Categ_Q.py b/bin/ZMQ_PubSub_Categ_Q.py index 7780dd2f..08a93e2f 100755 --- a/bin/ZMQ_PubSub_Categ_Q.py +++ b/bin/ZMQ_PubSub_Categ_Q.py @@ -24,6 +24,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = 'Queuing' config_section = 'PubSub_Words' diff --git a/bin/ZMQ_PubSub_Lines.py b/bin/ZMQ_PubSub_Lines.py index 8d2d8cf3..abef2e95 100755 --- a/bin/ZMQ_PubSub_Lines.py +++ b/bin/ZMQ_PubSub_Lines.py @@ -35,6 +35,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Global' diff --git a/bin/ZMQ_PubSub_Lines_Q.py b/bin/ZMQ_PubSub_Lines_Q.py index abf1fd35..3e366d82 100755 --- a/bin/ZMQ_PubSub_Lines_Q.py +++ b/bin/ZMQ_PubSub_Lines_Q.py @@ -23,6 +23,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = "PubSub_Global" diff --git a/bin/ZMQ_PubSub_Tokenize.py b/bin/ZMQ_PubSub_Tokenize.py index 2fe3de5b..7e1f45e1 100755 --- a/bin/ZMQ_PubSub_Tokenize.py +++ b/bin/ZMQ_PubSub_Tokenize.py @@ -30,6 +30,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Longlines' diff --git a/bin/ZMQ_PubSub_Tokenize_Q.py b/bin/ZMQ_PubSub_Tokenize_Q.py index 7594c37a..459ef88e 100755 --- a/bin/ZMQ_PubSub_Tokenize_Q.py +++ b/bin/ZMQ_PubSub_Tokenize_Q.py @@ -24,6 +24,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Longlines' diff --git a/bin/ZMQ_Pub_Global.py b/bin/ZMQ_Pub_Global.py index 7ac5ede7..2e687432 100755 --- a/bin/ZMQ_Pub_Global.py +++ b/bin/ZMQ_Pub_Global.py @@ -27,6 +27,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Global" config_section = 'PubSub_Global' diff --git a/bin/ZMQ_Sub_Attributes.py b/bin/ZMQ_Sub_Attributes.py index 9b48b2be..912f40e6 100755 --- a/bin/ZMQ_Sub_Attributes.py +++ b/bin/ZMQ_Sub_Attributes.py @@ -33,6 +33,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Global' diff --git a/bin/ZMQ_Sub_Attributes_Q.py b/bin/ZMQ_Sub_Attributes_Q.py index 6117174d..5d1a349f 100755 --- a/bin/ZMQ_Sub_Attributes_Q.py +++ b/bin/ZMQ_Sub_Attributes_Q.py @@ -24,6 +24,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Global' diff --git a/bin/ZMQ_Sub_CreditCards.py b/bin/ZMQ_Sub_CreditCards.py index 99db6692..22a8c5c5 100755 --- a/bin/ZMQ_Sub_CreditCards.py +++ b/bin/ZMQ_Sub_CreditCards.py @@ -9,6 +9,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_CreditCards_Q.py b/bin/ZMQ_Sub_CreditCards_Q.py index cec6f421..81a79c7c 100755 --- a/bin/ZMQ_Sub_CreditCards_Q.py +++ b/bin/ZMQ_Sub_CreditCards_Q.py @@ -7,6 +7,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Curve.py b/bin/ZMQ_Sub_Curve.py index acb2a167..c62ae32c 100755 --- a/bin/ZMQ_Sub_Curve.py +++ b/bin/ZMQ_Sub_Curve.py @@ -32,6 +32,7 @@ import os import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Words' @@ -53,9 +54,9 @@ if __name__ == "__main__": publisher.info("Script Curve subscribed to {}".format(h.sub_channel)) # FILE CURVE SECTION # - csv_path = os.path.join(os.environ('AIL_HOME'), + csv_path = os.path.join(os.environ['AIL_HOME'], h.config.get("Directories", "wordtrending_csv")) - wordfile_path = os.path.join(os.environ('AIL_HOME'), + wordfile_path = os.path.join(os.environ['AIL_HOME'], h.config.get("Directories", "wordsfile")) message = h.redis_rpop() diff --git a/bin/ZMQ_Sub_Curve_Q.py b/bin/ZMQ_Sub_Curve_Q.py index 572c9c79..b7264639 100755 --- a/bin/ZMQ_Sub_Curve_Q.py +++ b/bin/ZMQ_Sub_Curve_Q.py @@ -24,6 +24,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Words' diff --git a/bin/ZMQ_Sub_Duplicate.py b/bin/ZMQ_Sub_Duplicate.py index f0833e5e..eca87934 100755 --- a/bin/ZMQ_Sub_Duplicate.py +++ b/bin/ZMQ_Sub_Duplicate.py @@ -22,6 +22,7 @@ from pybloomfilter import BloomFilter import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Global' @@ -48,7 +49,7 @@ if __name__ == "__main__": h.config.get("PubSub_Global", "channel"))) set_limit = 100 - bloompath = os.path.join(os.environ('AIL_HOME'), + bloompath = os.path.join(os.environ['AIL_HOME'], h.config.get("Directories", "bloomfilters")) bloop_path_set = set() diff --git a/bin/ZMQ_Sub_Duplicate_Q.py b/bin/ZMQ_Sub_Duplicate_Q.py index 24af023e..9fc455ba 100755 --- a/bin/ZMQ_Sub_Duplicate_Q.py +++ b/bin/ZMQ_Sub_Duplicate_Q.py @@ -6,6 +6,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = 'Queuing' config_section = 'PubSub_Global' diff --git a/bin/ZMQ_Sub_Indexer.py b/bin/ZMQ_Sub_Indexer.py index 6fa4b233..0eb6dcce 100755 --- a/bin/ZMQ_Sub_Indexer.py +++ b/bin/ZMQ_Sub_Indexer.py @@ -21,6 +21,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" # Subscriber diff --git a/bin/ZMQ_Sub_Indexer_Q.py b/bin/ZMQ_Sub_Indexer_Q.py index cbdb4a05..af7e076a 100755 --- a/bin/ZMQ_Sub_Indexer_Q.py +++ b/bin/ZMQ_Sub_Indexer_Q.py @@ -18,6 +18,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Global' diff --git a/bin/ZMQ_Sub_Mails.py b/bin/ZMQ_Sub_Mails.py index ef91600a..e3c7e91a 100755 --- a/bin/ZMQ_Sub_Mails.py +++ b/bin/ZMQ_Sub_Mails.py @@ -12,6 +12,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Mails_Q.py b/bin/ZMQ_Sub_Mails_Q.py index 0f58fb7d..7a6e66e9 100755 --- a/bin/ZMQ_Sub_Mails_Q.py +++ b/bin/ZMQ_Sub_Mails_Q.py @@ -6,6 +6,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Onion.py b/bin/ZMQ_Sub_Onion.py index 58a9d38b..2bf27c85 100755 --- a/bin/ZMQ_Sub_Onion.py +++ b/bin/ZMQ_Sub_Onion.py @@ -30,6 +30,7 @@ from pubsublogger import publisher import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Onion_Q.py b/bin/ZMQ_Sub_Onion_Q.py index c979e104..6cbc9938 100755 --- a/bin/ZMQ_Sub_Onion_Q.py +++ b/bin/ZMQ_Sub_Onion_Q.py @@ -23,6 +23,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Urls.py b/bin/ZMQ_Sub_Urls.py index bfe282fc..b34de8fb 100755 --- a/bin/ZMQ_Sub_Urls.py +++ b/bin/ZMQ_Sub_Urls.py @@ -17,6 +17,7 @@ import ipaddress import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Script" config_section = 'PubSub_Categ' diff --git a/bin/ZMQ_Sub_Urls_Q.py b/bin/ZMQ_Sub_Urls_Q.py index 4f264d9a..1e63ae8c 100755 --- a/bin/ZMQ_Sub_Urls_Q.py +++ b/bin/ZMQ_Sub_Urls_Q.py @@ -7,6 +7,7 @@ import Helper if __name__ == "__main__": + publisher.port = 6380 publisher.channel = "Queuing" config_section = 'PubSub_Categ' diff --git a/bin/indexer_lookup.py b/bin/indexer_lookup.py index 3a239f13..b8fc6e1a 100644 --- a/bin/indexer_lookup.py +++ b/bin/indexer_lookup.py @@ -22,12 +22,12 @@ def readdoc(path=None): f = gzip.open(path, 'r') return f.read() -configfile = os.path.join(os.environ('AIL_BIN'), 'packages/config.cfg') +configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') cfg = ConfigParser.ConfigParser() cfg.read(configfile) # Indexer configuration - index dir and schema setup -indexpath = os.path.join(os.environ('AIL_HOME'), cfg.get("Indexer", "path")) +indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) indexertype = cfg.get("Indexer", "type") argParser = argparse.ArgumentParser(description='Fulltext search for AIL') diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 564dcc7d..de02b887 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -56,7 +56,7 @@ class Paste(object): def __init__(self, p_path): - configfile = os.path.join(os.environ('AIL_BIN'), 'packages/config.cfg') + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \