python 3 backend upgrade

This commit is contained in:
Terrtia 2018-04-16 14:50:04 +02:00
parent 3395b16873
commit 9c82dd90ec
54 changed files with 445 additions and 370 deletions

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -33,7 +33,7 @@ if __name__ == "__main__":
PST = Paste.Paste(message) PST = Paste.Paste(message)
else: else:
publisher.debug("Script Attribute is idling 1s") publisher.debug("Script Attribute is idling 1s")
print 'sleeping' print('sleeping')
time.sleep(1) time.sleep(1)
continue continue
@ -45,6 +45,6 @@ if __name__ == "__main__":
# FIXME Not used. # FIXME Not used.
PST.store.sadd("Pastes_Objects", PST.p_path) PST.store.sadd("Pastes_Objects", PST.p_path)
except IOError: except IOError:
print "CRC Checksum Failed on :", PST.p_path print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name)) PST.p_source, PST.p_date, PST.p_name))

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_PubSub_Categ Module The ZMQ_PubSub_Categ Module
@ -73,7 +73,7 @@ if __name__ == "__main__":
bname = os.path.basename(filename) bname = os.path.basename(filename)
tmp_dict[bname] = [] tmp_dict[bname] = []
with open(os.path.join(args.d, filename), 'r') as f: with open(os.path.join(args.d, filename), 'r') as f:
patterns = [r'%s' % re.escape(s.strip()) for s in f] patterns = [r'%s' % ( re.escape(s.strip()) ) for s in f]
tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE) tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE)
prec_filename = None prec_filename = None
@ -82,18 +82,25 @@ if __name__ == "__main__":
filename = p.get_from_set() filename = p.get_from_set()
if filename is None: if filename is None:
publisher.debug("Script Categ is Idling 10s") publisher.debug("Script Categ is Idling 10s")
print 'Sleeping' print('Sleeping')
time.sleep(10) time.sleep(10)
continue continue
paste = Paste.Paste(filename) paste = Paste.Paste(filename)
content = paste.get_p_content() content = paste.get_p_content()
print('-----------------------------------------------------')
print(filename)
print(content)
print('-----------------------------------------------------')
for categ, pattern in tmp_dict.items(): for categ, pattern in tmp_dict.items():
found = set(re.findall(pattern, content)) found = set(re.findall(pattern, content))
if len(found) >= matchingThreshold: if len(found) >= matchingThreshold:
msg = '{} {}'.format(paste.p_path, len(found)) msg = '{} {}'.format(paste.p_path, len(found))
print msg, categ #msg = " ".join( [paste.p_path, bytes(len(found))] )
print(msg, categ)
p.populate_set_out(msg, categ) p.populate_set_out(msg, categ)
publisher.info( publisher.info(

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -48,7 +48,7 @@ if __name__ == "__main__":
config_section = "Credential" config_section = "Credential"
p = Process(config_section) p = Process(config_section)
publisher.info("Find credentials") publisher.info("Find credentials")
minimumLengthThreshold = p.config.getint("Credential", "minimumLengthThreshold") minimumLengthThreshold = p.config.getint("Credential", "minimumLengthThreshold")
faup = Faup() faup = Faup()
@ -61,26 +61,34 @@ if __name__ == "__main__":
minTopPassList = p.config.getint("Credential", "minTopPassList") minTopPassList = p.config.getint("Credential", "minTopPassList")
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+" #regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
regex_cred = "[a-zA-Z0-9\\._-]+@[a-zA-Z0-9\\.-]+\.[a-zA-Z]{2,6}[\\rn :\_\-]{1,10}[a-zA-Z0-9\_\-]+"
regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:" regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:"
while True: while True:
message = p.get_from_set() message = p.get_from_set()
if message is None: if message is None:
publisher.debug("Script Credential is Idling 10s") publisher.debug("Script Credential is Idling 10s")
print('sleeping 10s') print('sleeping 10s')
time.sleep(10) time.sleep(1)
continue continue
filepath, count = message.split() filepath, count = message.split(' ')
if count < minTopPassList: #if count < minTopPassList:
# Less than 5 matches from the top password list, false positive. # Less than 5 matches from the top password list, false positive.
print("false positive:", count) #print("false positive:", count)
continue #continue
paste = Paste.Paste(filepath) paste = Paste.Paste(filepath)
content = paste.get_p_content() content = paste.get_p_content()
creds = set(re.findall(regex_cred, content)) creds = set(re.findall(regex_cred, content))
print(len(creds))
print(creds)
print(content)
print('-----')
publisher.warning('to_print')
if len(creds) == 0: if len(creds) == 0:
continue continue
@ -89,7 +97,7 @@ if __name__ == "__main__":
message = 'Checked {} credentials found.'.format(len(creds)) message = 'Checked {} credentials found.'.format(len(creds))
if sites_set: if sites_set:
message += ' Related websites: {}'.format(', '.join(sites_set)) message += ' Related websites: {}'.format( (', '.join(sites_set)) )
to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path) to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path)
@ -97,13 +105,14 @@ if __name__ == "__main__":
#num of creds above tresh, publish an alert #num of creds above tresh, publish an alert
if len(creds) > criticalNumberToAlert: if len(creds) > criticalNumberToAlert:
print("========> Found more than 10 credentials in this file : {}".format(filepath)) print("========> Found more than 10 credentials in this file : {}".format( filepath ))
publisher.warning(to_print) publisher.warning(to_print)
#Send to duplicate #Send to duplicate
p.populate_set_out(filepath, 'Duplicate') p.populate_set_out(filepath, 'Duplicate')
#Send to alertHandler #Send to alertHandler
p.populate_set_out('credential;{}'.format(filepath), 'alertHandler') msg = 'credential;{}'.format(filepath)
p.populate_set_out(msg, 'alertHandler')
#Put in form, count occurences, then send to moduleStats #Put in form, count occurences, then send to moduleStats
creds_sites = {} creds_sites = {}
site_occurence = re.findall(regex_site_for_stats, content) site_occurence = re.findall(regex_site_for_stats, content)
@ -122,9 +131,10 @@ if __name__ == "__main__":
else: else:
creds_sites[domain] = 1 creds_sites[domain] = 1
for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats for site, num in creds_sites.items(): # Send for each different site to moduleStats
print 'credential;{};{};{}'.format(num, site, paste.p_date) mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
p.populate_set_out('credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats') print(mssg)
p.populate_set_out(mssg, 'ModuleStats')
if sites_set: if sites_set:
print("=======> Probably on : {}".format(', '.join(sites_set))) print("=======> Probably on : {}".format(', '.join(sites_set)))
@ -148,7 +158,7 @@ if __name__ == "__main__":
uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME) uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME)
server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred}) server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred}) server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
#Add the mapping between the credential and the path #Add the mapping between the credential and the path
server_cred.sadd(REDIS_KEY_MAP_CRED_TO_PATH+'_'+str(uniq_num_cred), uniq_num_path) server_cred.sadd(REDIS_KEY_MAP_CRED_TO_PATH+'_'+str(uniq_num_cred), uniq_num_path)
@ -158,4 +168,3 @@ if __name__ == "__main__":
for partCred in splitedCred: for partCred in splitedCred:
if len(partCred) > minimumLengthThreshold: if len(partCred) > minimumLengthThreshold:
server_cred.sadd(partCred, uniq_num_cred) server_cred.sadd(partCred, uniq_num_cred)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -18,6 +18,7 @@ from packages import Paste
from packages import lib_refine from packages import lib_refine
from pubsublogger import publisher from pubsublogger import publisher
import re import re
import sys
from Helper import Process from Helper import Process
@ -58,13 +59,14 @@ if __name__ == "__main__":
content = paste.get_p_content() content = paste.get_p_content()
all_cards = re.findall(regex, content) all_cards = re.findall(regex, content)
if len(all_cards) > 0: if len(all_cards) > 0:
print 'All matching', all_cards print('All matching', all_cards)
creditcard_set = set([]) creditcard_set = set([])
for card in all_cards: for card in all_cards:
clean_card = re.sub('[^0-9]', '', card) clean_card = re.sub('[^0-9]', '', card)
clean_card = clean_card
if lib_refine.is_luhn_valid(clean_card): if lib_refine.is_luhn_valid(clean_card):
print clean_card, 'is valid' print(clean_card, 'is valid')
creditcard_set.add(clean_card) creditcard_set.add(clean_card)
paste.__setattr__(channel, creditcard_set) paste.__setattr__(channel, creditcard_set)
@ -76,13 +78,15 @@ if __name__ == "__main__":
if (len(creditcard_set) > 0): if (len(creditcard_set) > 0):
publisher.warning('{}Checked {} valid number(s);{}'.format( publisher.warning('{}Checked {} valid number(s);{}'.format(
to_print, len(creditcard_set), paste.p_path)) to_print, len(creditcard_set), paste.p_path))
print('{}Checked {} valid number(s);{}'.format(
to_print, len(creditcard_set), paste.p_path))
#Send to duplicate #Send to duplicate
p.populate_set_out(filename, 'Duplicate') p.populate_set_out(filename, 'Duplicate')
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('creditcard;{}'.format(filename), 'alertHandler') msg = 'creditcard;{}'.format(filename)
p.populate_set_out(msg, 'alertHandler')
else: else:
publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path)) publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
else: else:
publisher.debug("Script creditcard is idling 1m") publisher.debug("Script creditcard is idling 1m")
time.sleep(10) time.sleep(10)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module. This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
@ -53,12 +53,12 @@ def check_if_tracked_term(term, path):
#add_paste to tracked_word_set #add_paste to tracked_word_set
set_name = "tracked_" + term set_name = "tracked_" + term
server_term.sadd(set_name, path) server_term.sadd(set_name, path)
print term, 'addded', set_name, '->', path print(term, 'addded', set_name, '->', path)
p.populate_set_out("New Term added", 'CurveManageTopSets') p.populate_set_out("New Term added", 'CurveManageTopSets')
# Send a notification only when the member is in the set # Send a notification only when the member is in the set
if term in server_term.smembers(TrackedTermsNotificationEnabled_Name): if term in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress # Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, term) sendEmailNotification(email, term)
@ -137,7 +137,7 @@ if __name__ == "__main__":
server_term.zincrby(curr_set, low_word, float(score)) server_term.zincrby(curr_set, low_word, float(score))
#1 term per paste #1 term per paste
server_term.zincrby("per_paste_" + curr_set, low_word, float(1)) server_term.zincrby("per_paste_" + curr_set, low_word, float(1))
#Add more info for tracked terms #Add more info for tracked terms
check_if_tracked_term(low_word, filename) check_if_tracked_term(low_word, filename)
@ -149,15 +149,16 @@ if __name__ == "__main__":
if generate_new_graph: if generate_new_graph:
generate_new_graph = False generate_new_graph = False
print 'Building graph' print('Building graph')
today = datetime.date.today() today = datetime.date.today()
year = today.year year = today.year
month = today.month month = today.month
lib_words.create_curve_with_word_file(r_serv1, csv_path, lib_words.create_curve_with_word_file(r_serv1, csv_path,
wordfile_path, year, wordfile_path, year,
month) month)
publisher.debug("Script Curve is Idling") publisher.debug("Script Curve is Idling")
print "sleeping" print("sleeping")
time.sleep(10) time.sleep(10)
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -16,7 +16,7 @@ from packages import lib_words
import datetime import datetime
import calendar import calendar
import os import os
import ConfigParser import configparser
# Config Variables # Config Variables
Refresh_rate = 60*5 #sec Refresh_rate = 60*5 #sec
@ -68,26 +68,26 @@ def manage_top_set():
# convert dico into sorted array # convert dico into sorted array
array_month = [] array_month = []
for w, v in dico.iteritems(): for w, v in dico.items():
array_month.append((w, v)) array_month.append((w, v))
array_month.sort(key=lambda tup: -tup[1]) array_month.sort(key=lambda tup: -tup[1])
array_month = array_month[0:20] array_month = array_month[0:20]
array_week = [] array_week = []
for w, v in dico_week.iteritems(): for w, v in dico_week.items():
array_week.append((w, v)) array_week.append((w, v))
array_week.sort(key=lambda tup: -tup[1]) array_week.sort(key=lambda tup: -tup[1])
array_week = array_week[0:20] array_week = array_week[0:20]
# convert dico_per_paste into sorted array # convert dico_per_paste into sorted array
array_month_per_paste = [] array_month_per_paste = []
for w, v in dico_per_paste.iteritems(): for w, v in dico_per_paste.items():
array_month_per_paste.append((w, v)) array_month_per_paste.append((w, v))
array_month_per_paste.sort(key=lambda tup: -tup[1]) array_month_per_paste.sort(key=lambda tup: -tup[1])
array_month_per_paste = array_month_per_paste[0:20] array_month_per_paste = array_month_per_paste[0:20]
array_week_per_paste = [] array_week_per_paste = []
for w, v in dico_week_per_paste.iteritems(): for w, v in dico_week_per_paste.items():
array_week_per_paste.append((w, v)) array_week_per_paste.append((w, v))
array_week_per_paste.sort(key=lambda tup: -tup[1]) array_week_per_paste.sort(key=lambda tup: -tup[1])
array_week_per_paste = array_week_per_paste[0:20] array_week_per_paste = array_week_per_paste[0:20]
@ -105,7 +105,7 @@ def manage_top_set():
server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0]) server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0])
for elem in array_week_per_paste: for elem in array_week_per_paste:
server_term.zadd("per_paste_" + top_termFreq_setName_week[0], float(elem[1]), elem[0]) server_term.zadd("per_paste_" + top_termFreq_setName_week[0], float(elem[1]), elem[0])
for elem in array_month: for elem in array_month:
server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0]) server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0])
for elem in array_month_per_paste: for elem in array_month_per_paste:
@ -114,7 +114,7 @@ def manage_top_set():
timestamp = int(time.mktime(datetime.datetime.now().timetuple())) timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
value = str(timestamp) + ", " + "-" value = str(timestamp) + ", " + "-"
r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value) r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value)
print "refreshed module" print("refreshed module")
@ -130,8 +130,8 @@ if __name__ == '__main__':
raise Exception('Unable to find the configuration file. \ raise Exception('Unable to find the configuration file. \
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
@ -162,4 +162,3 @@ if __name__ == '__main__':
# Get one message from the input queue (module only work if linked with a queue) # Get one message from the input queue (module only work if linked with a queue)
time.sleep(Refresh_rate) # sleep a long time then manage the set time.sleep(Refresh_rate) # sleep a long time then manage the set
manage_top_set() manage_top_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The CVE Module The CVE Module
@ -32,7 +32,8 @@ def search_cve(message):
publisher.warning('{} contains CVEs'.format(paste.p_name)) publisher.warning('{} contains CVEs'.format(paste.p_name))
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('cve;{}'.format(filepath), 'alertHandler') msg = 'cve;{}'.format(filepath)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate #Send to duplicate
p.populate_set_out(filepath, 'Duplicate') p.populate_set_out(filepath, 'Duplicate')
@ -63,4 +64,3 @@ if __name__ == '__main__':
# Do something with the message from the queue # Do something with the message from the queue
search_cve(message) search_cve(message)

View file

@ -1,18 +1,18 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import argparse import argparse
import redis import redis
from pubsublogger import publisher from pubsublogger import publisher
from packages.lib_words import create_dirfile from packages.lib_words import create_dirfile
import ConfigParser import configparser
def main(): def main():
"""Main Function""" """Main Function"""
# CONFIG # # CONFIG #
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read('./packages/config.cfg') cfg.read('./packages/config.cfg')
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(

View file

@ -1,11 +1,11 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The DomClassifier Module The DomClassifier Module
============================ ============================
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
the out output of the Global module. the out output of the Global module.
""" """
@ -44,6 +44,7 @@ def main():
continue continue
paste = PST.get_p_content() paste = PST.get_p_content()
mimetype = PST._get_p_encoding() mimetype = PST._get_p_encoding()
if mimetype == "text/plain": if mimetype == "text/plain":
c.text(rawtext=paste) c.text(rawtext=paste)
c.potentialdomain() c.potentialdomain()
@ -59,7 +60,7 @@ def main():
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
except IOError: except IOError:
print "CRC Checksum Failed on :", PST.p_path print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name)) PST.p_source, PST.p_date, PST.p_name))

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
from pubsublogger import publisher from pubsublogger import publisher
@ -23,7 +23,7 @@ if __name__ == "__main__":
if message is not None: if message is not None:
f = open(dump_file, 'a') f = open(dump_file, 'a')
while message is not None: while message is not None:
print message print(message)
date = datetime.datetime.now() date = datetime.datetime.now()
if message is not None: if message is not None:
f.write(date.isoformat() + ' ' + message + '\n') f.write(date.isoformat() + ' ' + message + '\n')

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -42,15 +42,15 @@ if __name__ == "__main__":
threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep")) threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep"))
threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh")) threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh"))
threshold_set = {} threshold_set = {}
threshold_set['ssdeep'] = threshold_duplicate_ssdeep threshold_set['ssdeep'] = threshold_duplicate_ssdeep
threshold_set['tlsh'] = threshold_duplicate_tlsh threshold_set['tlsh'] = threshold_duplicate_tlsh
min_paste_size = float(p.config.get("Modules_Duplicates", "min_paste_size")) min_paste_size = float(p.config.get("Modules_Duplicates", "min_paste_size"))
# REDIS # # REDIS #
dico_redis = {} dico_redis = {}
date_today = datetime.today() date_today = datetime.today()
for year in xrange(2013, date_today.year+1): for year in range(2013, date_today.year+1):
for month in xrange(0, 13): for month in range(0, 13):
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"), port=year, host=p.config.get("Redis_Level_DB", "host"), port=year,
db=month) db=month)
@ -90,7 +90,7 @@ if __name__ == "__main__":
# Get the date of the range # Get the date of the range
date_range = date_today - timedelta(days = maximum_month_range*30.4166666) date_range = date_today - timedelta(days = maximum_month_range*30.4166666)
num_of_month = (date_today.year - date_range.year)*12 + (date_today.month - date_range.month) num_of_month = (date_today.year - date_range.year)*12 + (date_today.month - date_range.month)
for diff_month in xrange(0, num_of_month+1): for diff_month in range(0, num_of_month+1):
curr_date_range = date_today - timedelta(days = diff_month*30.4166666) curr_date_range = date_today - timedelta(days = diff_month*30.4166666)
to_append = str(curr_date_range.year)+str(curr_date_range.month).zfill(2) to_append = str(curr_date_range.year)+str(curr_date_range.month).zfill(2)
dico_range_list.append(to_append) dico_range_list.append(to_append)
@ -102,7 +102,7 @@ if __name__ == "__main__":
yearly_index = str(date_today.year)+'00' yearly_index = str(date_today.year)+'00'
r_serv0 = dico_redis[yearly_index] r_serv0 = dico_redis[yearly_index]
r_serv0.incr("current_index") r_serv0.incr("current_index")
index = r_serv0.get("current_index")+str(PST.p_date) index = (r_serv0.get("current_index")).decode('utf8') + str(PST.p_date)
# Open selected dico range # Open selected dico range
opened_dico = [] opened_dico = []
@ -114,11 +114,13 @@ if __name__ == "__main__":
# Go throught the Database of the dico (of the month) # Go throught the Database of the dico (of the month)
for curr_dico_name, curr_dico_redis in opened_dico: for curr_dico_name, curr_dico_redis in opened_dico:
for hash_type, paste_hash in paste_hashes.iteritems(): for hash_type, paste_hash in paste_hashes.items():
for dico_hash in curr_dico_redis.smembers('HASHS_'+hash_type): for dico_hash in curr_dico_redis.smembers('HASHS_'+hash_type):
dico_hash = dico_hash.decode('utf8')
try: try:
if hash_type == 'ssdeep': if hash_type == 'ssdeep':
percent = 100-ssdeep.compare(dico_hash, paste_hash) percent = 100-ssdeep.compare(dico_hash, paste_hash)
else: else:
percent = tlsh.diffxlen(dico_hash, paste_hash) percent = tlsh.diffxlen(dico_hash, paste_hash)
@ -130,15 +132,18 @@ if __name__ == "__main__":
# index of paste # index of paste
index_current = r_serv_dico.get(dico_hash) index_current = r_serv_dico.get(dico_hash)
index_current = index_current.decode('utf8')
paste_path = r_serv_dico.get(index_current) paste_path = r_serv_dico.get(index_current)
paste_path = paste_path.decode('utf8')
paste_date = r_serv_dico.get(index_current+'_date') paste_date = r_serv_dico.get(index_current+'_date')
paste_date = paste_date.decode('utf8')
paste_date = paste_date if paste_date != None else "No date available" paste_date = paste_date if paste_date != None else "No date available"
if paste_path != None: if paste_path != None:
hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date) hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent) print('['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent))
except Exception,e: except Exception:
print str(e) print(str(e))
#print 'hash not comparable, bad hash: '+dico_hash+' , current_hash: '+paste_hash #print 'hash not comparable, bad hash: '+dico_hash+' , current_hash: '+paste_hash
# Add paste in DB after checking to prevent its analysis twice # Add paste in DB after checking to prevent its analysis twice
@ -147,7 +152,7 @@ if __name__ == "__main__":
r_serv1.set(index+'_date', PST._get_p_date()) r_serv1.set(index+'_date', PST._get_p_date())
r_serv1.sadd("INDEX", index) r_serv1.sadd("INDEX", index)
# Adding hashes in Redis # Adding hashes in Redis
for hash_type, paste_hash in paste_hashes.iteritems(): for hash_type, paste_hash in paste_hashes.items():
r_serv1.set(paste_hash, index) r_serv1.set(paste_hash, index)
r_serv1.sadd("HASHS_"+hash_type, paste_hash) r_serv1.sadd("HASHS_"+hash_type, paste_hash)
@ -166,7 +171,7 @@ if __name__ == "__main__":
PST.__setattr__("p_duplicate", dupl) PST.__setattr__("p_duplicate", dupl)
PST.save_attribute_redis("p_duplicate", dupl) PST.save_attribute_redis("p_duplicate", dupl)
publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path)) publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path))
print '{}Detected {}'.format(to_print, len(dupl)) print('{}Detected {}'.format(to_print, len(dupl)))
y = time.time() y = time.time()
@ -176,5 +181,5 @@ if __name__ == "__main__":
except IOError: except IOError:
to_print = 'Duplicate;{};{};{};'.format( to_print = 'Duplicate;{};{};{};'.format(
PST.p_source, PST.p_date, PST.p_name) PST.p_source, PST.p_date, PST.p_name)
print "CRC Checksum Failed on :", PST.p_path print("CRC Checksum Failed on :", PST.p_path)
publisher.error('{}CRC Checksum Failed'.format(to_print)) publisher.error('{}CRC Checksum Failed'.format(to_print))

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -33,8 +33,8 @@ if __name__ == "__main__":
# DB OBJECT & HASHS ( DISK ) # DB OBJECT & HASHS ( DISK )
# FIXME increase flexibility # FIXME increase flexibility
dico_redis = {} dico_redis = {}
for year in xrange(2013, 2017): for year in range(2013, 2017):
for month in xrange(0, 16): for month in range(0, 16):
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"), port=year, host=p.config.get("Redis_Level_DB", "host"), port=year,
db=month) db=month)
@ -147,7 +147,7 @@ if __name__ == "__main__":
if percentage >= 50: if percentage >= 50:
dupl.append((paste, percentage)) dupl.append((paste, percentage))
else: else:
print 'percentage: ' + str(percentage) print('percentage: ' + str(percentage))
# Creating the object attribute and save it. # Creating the object attribute and save it.
to_print = 'Duplicate;{};{};{};'.format( to_print = 'Duplicate;{};{};{};'.format(
@ -156,11 +156,11 @@ if __name__ == "__main__":
PST.__setattr__("p_duplicate", dupl) PST.__setattr__("p_duplicate", dupl)
PST.save_attribute_redis("p_duplicate", dupl) PST.save_attribute_redis("p_duplicate", dupl)
publisher.info('{}Detected {}'.format(to_print, len(dupl))) publisher.info('{}Detected {}'.format(to_print, len(dupl)))
print '{}Detected {}'.format(to_print, len(dupl)) print('{}Detected {}'.format(to_print, len(dupl)))
y = time.time() y = time.time()
publisher.debug('{}Processed in {} sec'.format(to_print, y-x)) publisher.debug('{}Processed in {} sec'.format(to_print, y-x))
except IOError: except IOError:
print "CRC Checksum Failed on :", PST.p_path print("CRC Checksum Failed on :", PST.p_path)
publisher.error('{}CRC Checksum Failed'.format(to_print)) publisher.error('{}CRC Checksum Failed'.format(to_print))

View file

@ -67,7 +67,7 @@ if __name__ == '__main__':
continue continue
# Creating the full filepath # Creating the full filepath
filename = os.path.join(os.environ['AIL_HOME'], filename = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), paste.decode('utf8')) p.config.get("Directories", "pastes"), paste)
#print(filename) #print(filename)
dirname = os.path.dirname(filename) dirname = os.path.dirname(filename)
if not os.path.exists(dirname): if not os.path.exists(dirname):
@ -77,5 +77,5 @@ if __name__ == '__main__':
f.write(base64.standard_b64decode(gzip64encoded)) f.write(base64.standard_b64decode(gzip64encoded))
print(filename) print(filename)
p.populate_set_out(filename.encode('utf8')) p.populate_set_out(filename)
processed_paste+=1 processed_paste+=1

View file

@ -58,7 +58,7 @@ class PubSub(object):
new_sub = context.socket(zmq.SUB) new_sub = context.socket(zmq.SUB)
new_sub.connect(address) new_sub.connect(address)
# bytes64 encode bytes to ascii only bytes # bytes64 encode bytes to ascii only bytes
new_sub.setsockopt(zmq.SUBSCRIBE, channel.encode('ascii')) new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
self.subscribers.append(new_sub) self.subscribers.append(new_sub)
def setup_publish(self, conn_name): def setup_publish(self, conn_name):
@ -78,15 +78,15 @@ class PubSub(object):
self.publishers['ZMQ'].append((p, channel)) self.publishers['ZMQ'].append((p, channel))
def publish(self, message): def publish(self, message):
m = json.loads(message.decode('ascii')) m = json.loads(message)
channel_message = m.get('channel') channel_message = m.get('channel')
for p, channel in self.publishers['Redis']: for p, channel in self.publishers['Redis']:
if channel_message is None or channel_message == channel: if channel_message is None or channel_message == channel:
p.publish(channel, ( m['message']).encode('ascii') ) p.publish(channel, ( m['message']) )
for p, channel in self.publishers['ZMQ']: for p, channel in self.publishers['ZMQ']:
if channel_message is None or channel_message == channel: if channel_message is None or channel_message == channel:
mess = ( m['message'] ).encode('ascii') p.send('{} {}'.format(channel, m['message']))
p.send(b' '.join( [channel, mess] ) ) #p.send(b' '.join( [channel, mess] ) )
def subscribe(self): def subscribe(self):
@ -99,7 +99,8 @@ class PubSub(object):
for sub in self.subscribers: for sub in self.subscribers:
try: try:
msg = sub.recv(zmq.NOBLOCK) msg = sub.recv(zmq.NOBLOCK)
yield msg.split(b" ", 1)[1] msg = msg.decode('utf8')
yield msg.split(" ", 1)[1]
except zmq.error.Again as e: except zmq.error.Again as e:
time.sleep(0.2) time.sleep(0.2)
pass pass
@ -150,6 +151,12 @@ class Process(object):
self.r_temp.hset('queues', self.subscriber_name, self.r_temp.hset('queues', self.subscriber_name,
int(self.r_temp.scard(in_set))) int(self.r_temp.scard(in_set)))
message = self.r_temp.spop(in_set) message = self.r_temp.spop(in_set)
try:
message = message.decode('utf8')
except AttributeError:
pass
timestamp = int(time.mktime(datetime.datetime.now().timetuple())) timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
dir_name = os.environ['AIL_HOME']+self.config.get('Directories', 'pastes') dir_name = os.environ['AIL_HOME']+self.config.get('Directories', 'pastes')
@ -158,12 +165,12 @@ class Process(object):
else: else:
#try: #try:
if b'.gz' in message: if '.gz' in message:
path = message.split(b".")[-2].split(b"/")[-1] path = message.split(".")[-2].split("/")[-1]
#find start of path with AIL_HOME #find start of path with AIL_HOME
index_s = (message.decode('ascii')).find(os.environ['AIL_HOME']) index_s = message.find(os.environ['AIL_HOME'])
#Stop when .gz #Stop when .gz
index_e = message.find(b".gz")+3 index_e = message.find(".gz")+3
if(index_s == -1): if(index_s == -1):
complete_path = message[0:index_e] complete_path = message[0:index_e]
else: else:
@ -173,7 +180,7 @@ class Process(object):
path = "-" path = "-"
complete_path = "?" complete_path = "?"
value = str(timestamp) + ", " + path.decode('ascii') value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value) self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path) self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
@ -190,13 +197,12 @@ class Process(object):
def populate_set_out(self, msg, channel=None): def populate_set_out(self, msg, channel=None):
# multiproc # multiproc
msg = msg.decode('ascii')
msg = {'message': msg} msg = {'message': msg}
if channel is not None: if channel is not None:
msg.update({'channel': channel}) msg.update({'channel': channel})
# bytes64 encode bytes to ascii only bytes # bytes64 encode bytes to ascii only bytes
j = (json.dumps(msg)).encode('ascii') j = json.dumps(msg)
self.r_temp.sadd(self.subscriber_name + 'out', j) self.r_temp.sadd(self.subscriber_name + 'out', j)
def publish(self): def publish(self):
@ -209,6 +215,12 @@ class Process(object):
self.pubsub.setup_publish(name) self.pubsub.setup_publish(name)
while True: while True:
message = self.r_temp.spop(self.subscriber_name + 'out') message = self.r_temp.spop(self.subscriber_name + 'out')
try:
message = message.decode('utf8')
except AttributeError:
pass
if message is None: if message is None:
time.sleep(1) time.sleep(1)
continue continue

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -49,7 +49,7 @@ if __name__ == "__main__":
# Indexer configuration - index dir and schema setup # Indexer configuration - index dir and schema setup
baseindexpath = join(os.environ['AIL_HOME'], baseindexpath = join(os.environ['AIL_HOME'],
p.config.get("Indexer", "path")) p.config.get("Indexer", "path"))
indexRegister_path = join(os.environ['AIL_HOME'], indexRegister_path = join(os.environ['AIL_HOME'],
p.config.get("Indexer", "register")) p.config.get("Indexer", "register"))
indexertype = p.config.get("Indexer", "type") indexertype = p.config.get("Indexer", "type")
INDEX_SIZE_THRESHOLD = int(p.config.get("Indexer", "index_max_size")) INDEX_SIZE_THRESHOLD = int(p.config.get("Indexer", "index_max_size"))
@ -89,7 +89,7 @@ if __name__ == "__main__":
ix = create_in(indexpath, schema) ix = create_in(indexpath, schema)
else: else:
ix = open_dir(indexpath) ix = open_dir(indexpath)
last_refresh = time_now last_refresh = time_now
# LOGGING # # LOGGING #
@ -107,10 +107,11 @@ if __name__ == "__main__":
continue continue
docpath = message.split(" ", -1)[-1] docpath = message.split(" ", -1)[-1]
paste = PST.get_p_content() paste = PST.get_p_content()
print "Indexing - "+indexname+" :", docpath print("Indexing - " + indexname + " :", docpath)
if time.time() - last_refresh > TIME_WAIT: #avoid calculating the index's size at each message #avoid calculating the index's size at each message
if( time.time() - last_refresh > TIME_WAIT):
last_refresh = time.time() last_refresh = time.time()
if check_index_size(baseindexpath, indexname) >= INDEX_SIZE_THRESHOLD*(1000*1000): if check_index_size(baseindexpath, indexname) >= INDEX_SIZE_THRESHOLD*(1000*1000):
timestamp = int(time.time()) timestamp = int(time.time())
@ -128,11 +129,11 @@ if __name__ == "__main__":
if indexertype == "whoosh": if indexertype == "whoosh":
indexwriter = ix.writer() indexwriter = ix.writer()
indexwriter.update_document( indexwriter.update_document(
title=unicode(docpath, errors='ignore'), title=docpath,
path=unicode(docpath, errors='ignore'), path=docpath,
content=unicode(paste, errors='ignore')) content=paste)
indexwriter.commit() indexwriter.commit()
except IOError: except IOError:
print "CRC Checksum Failed on :", PST.p_path print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name)) PST.p_source, PST.p_date, PST.p_name))

View file

@ -15,46 +15,49 @@ RSA private key, certificate messages
import time import time
from pubsublogger import publisher from pubsublogger import publisher
from bin.packages import Paste #from bin.packages import Paste
from bin.Helper import Process #from bin.Helper import Process
from packages import Paste
from Helper import Process
def search_key(paste): def search_key(paste):
content = paste.get_p_content() content = paste.get_p_content()
find = False find = False
if b'-----BEGIN PGP MESSAGE-----' in content: if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name)) publisher.warning('{} has a PGP enc message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN CERTIFICATE-----' in content: if '-----BEGIN CERTIFICATE-----' in content:
publisher.warning('{} has a certificate message'.format(paste.p_name)) publisher.warning('{} has a certificate message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN RSA PRIVATE KEY-----' in content: if '-----BEGIN RSA PRIVATE KEY-----' in content:
publisher.warning('{} has a RSA private key message'.format(paste.p_name)) publisher.warning('{} has a RSA private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN PRIVATE KEY-----' in content: if '-----BEGIN PRIVATE KEY-----' in content:
publisher.warning('{} has a private key message'.format(paste.p_name)) publisher.warning('{} has a private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
publisher.warning('{} has an encrypted private key message'.format(paste.p_name)) publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content: if '-----BEGIN OPENSSH PRIVATE KEY-----' in content:
publisher.warning('{} has an openssh private key message'.format(paste.p_name)) publisher.warning('{} has an openssh private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN DSA PRIVATE KEY-----' in content: if '-----BEGIN DSA PRIVATE KEY-----' in content:
publisher.warning('{} has a dsa private key message'.format(paste.p_name)) publisher.warning('{} has a dsa private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN EC PRIVATE KEY-----' in content: if '-----BEGIN EC PRIVATE KEY-----' in content:
publisher.warning('{} has an ec private key message'.format(paste.p_name)) publisher.warning('{} has an ec private key message'.format(paste.p_name))
find = True find = True
if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content: if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
publisher.warning('{} has a pgp private key block message'.format(paste.p_name)) publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
find = True find = True
@ -63,7 +66,9 @@ def search_key(paste):
#Send to duplicate #Send to duplicate
p.populate_set_out(message, 'Duplicate') p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'alertHandler') msg = ('keys;{}'.format(message))
print(message)
p.populate_set_out( msg, 'alertHandler')
if __name__ == '__main__': if __name__ == '__main__':

View file

@ -130,8 +130,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x' screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x'
sleep 0.1 sleep 0.1
#screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
#sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x' screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'python3 Tokenize.py; read x' screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'python3 Tokenize.py; read x'
@ -142,8 +142,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x' screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1 sleep 0.1
#screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x' screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
#sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c './Credential.py; read x' screen -S "Script_AIL" -X screen -t "Credential" bash -c './Credential.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Curve" bash -c './Curve.py; read x' screen -S "Script_AIL" -X screen -t "Curve" bash -c './Curve.py; read x'
@ -168,8 +168,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
sleep 0.1 sleep 0.1
#screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
#sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -61,7 +61,7 @@ if __name__ == '__main__':
while True: while True:
try: try:
message = p.get_from_set() message = p.get_from_set()
print message print(message)
if message is not None: if message is not None:
PST = Paste.Paste(message) PST = Paste.Paste(message)
else: else:
@ -77,8 +77,8 @@ if __name__ == '__main__':
# FIXME Not used. # FIXME Not used.
PST.store.sadd("Pastes_Objects", PST.p_path) PST.store.sadd("Pastes_Objects", PST.p_path)
if lines_infos[1] < args.max: if lines_infos[1] < args.max:
p.populate_set_out(PST.p_path, 'LinesShort') p.populate_set_out( PST.p_path , 'LinesShort')
else: else:
p.populate_set_out(PST.p_path, 'LinesLong') p.populate_set_out( PST.p_path , 'LinesLong')
except IOError: except IOError:
print "CRC Checksum Error on : ", PST.p_path print("CRC Checksum Error on : ", PST.p_path)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -73,19 +73,19 @@ if __name__ == "__main__":
#Send to duplicate #Send to duplicate
p.populate_set_out(filename, 'Duplicate') p.populate_set_out(filename, 'Duplicate')
p.populate_set_out('mail;{}'.format(filename), 'alertHandler') p.populate_set_out('mail;{}'.format(filename), 'alertHandler')
else: else:
publisher.info(to_print) publisher.info(to_print)
#Send to ModuleStats #Send to ModuleStats
for mail in MX_values[1]: for mail in MX_values[1]:
print 'mail;{};{};{}'.format(1, mail, PST.p_date) print('mail;{};{};{}'.format(1, mail, PST.p_date))
p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats') p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats')
prec_filename = filename prec_filename = filename
else: else:
publisher.debug("Script Mails is Idling 10s") publisher.debug("Script Mails is Idling 10s")
print 'Sleeping' print('Sleeping')
time.sleep(10) time.sleep(10)
message = p.get_from_set() message = p.get_from_set()

View file

@ -91,7 +91,7 @@ if __name__ == '__main__':
complete_paste, gzip64encoded = splitted complete_paste, gzip64encoded = splitted
try: try:
feeder_name = ( complete_paste.replace(b"archive/",b"") ).split(b"/")[0] feeder_name = ( complete_paste.replace("archive/","") ).split("/")[0]
# TODO take real name ? # TODO take real name ?
paste_name = complete_paste paste_name = complete_paste
@ -109,9 +109,10 @@ if __name__ == '__main__':
processed_paste_per_feeder[feeder_name] = 1 processed_paste_per_feeder[feeder_name] = 1
duplicated_paste_per_feeder[feeder_name] = 0 duplicated_paste_per_feeder[feeder_name] = 0
relay_message = b" ".join( [paste_name, gzip64encoded] ) relay_message = "{0} {1}".format(paste_name, gzip64encoded)
#relay_message = b" ".join( [paste_name, gzip64encoded] )
digest = hashlib.sha1(gzip64encoded).hexdigest()
digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
# Avoid any duplicate coming from any sources # Avoid any duplicate coming from any sources
if operation_mode == 1: if operation_mode == 1:

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
''' '''
@ -20,7 +20,7 @@ import os
import signal import signal
import argparse import argparse
from subprocess import PIPE, Popen from subprocess import PIPE, Popen
import ConfigParser import configparser
import json import json
from terminaltables import AsciiTable from terminaltables import AsciiTable
import textwrap import textwrap
@ -51,7 +51,7 @@ last_refresh = 0
def getPid(module): def getPid(module):
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
for line in p.stdout: for line in p.stdout:
print line print(line)
splittedLine = line.split() splittedLine = line.split()
if 'python2' in splittedLine: if 'python2' in splittedLine:
return int(splittedLine[0]) return int(splittedLine[0])
@ -76,7 +76,7 @@ def cleanRedis():
flag_pid_valid = True flag_pid_valid = True
if not flag_pid_valid: if not flag_pid_valid:
print flag_pid_valid, 'cleaning', pid, 'in', k print(flag_pid_valid, 'cleaning', pid, 'in', k)
server.srem(k, pid) server.srem(k, pid)
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k]) printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k])
@ -85,11 +85,11 @@ def cleanRedis():
def kill_module(module, pid): def kill_module(module, pid):
print '' print('')
print '-> trying to kill module:', module print('-> trying to kill module:', module)
if pid is None: if pid is None:
print 'pid was None' print('pid was None')
printarrayGlob.insert(1, [0, module, pid, "PID was None"]) printarrayGlob.insert(1, [0, module, pid, "PID was None"])
printarrayGlob.pop() printarrayGlob.pop()
pid = getPid(module) pid = getPid(module)
@ -102,15 +102,15 @@ def kill_module(module, pid):
try: try:
os.kill(pid, signal.SIGUSR1) os.kill(pid, signal.SIGUSR1)
except OSError: except OSError:
print pid, 'already killed' print(pid, 'already killed')
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"]) printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"])
printarrayGlob.pop() printarrayGlob.pop()
return return
time.sleep(1) time.sleep(1)
if getPid(module) is None: if getPid(module) is None:
print module, 'has been killed' print(module, 'has been killed')
print 'restarting', module, '...' print('restarting', module, '...')
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Killed"]) printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
@ -119,7 +119,7 @@ def kill_module(module, pid):
printarrayGlob.pop() printarrayGlob.pop()
else: else:
print 'killing failed, retrying...' print('killing failed, retrying...')
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."]) printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."])
printarrayGlob.pop() printarrayGlob.pop()
@ -128,8 +128,8 @@ def kill_module(module, pid):
os.kill(pid, signal.SIGUSR1) os.kill(pid, signal.SIGUSR1)
time.sleep(1) time.sleep(1)
if getPid(module) is None: if getPid(module) is None:
print module, 'has been killed' print(module, 'has been killed')
print 'restarting', module, '...' print('restarting', module, '...')
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Killed"]) printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
@ -137,12 +137,12 @@ def kill_module(module, pid):
printarrayGlob.pop() printarrayGlob.pop()
printarrayGlob.pop() printarrayGlob.pop()
else: else:
print 'killing failed!' print('killing failed!')
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"]) printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"])
printarrayGlob.pop() printarrayGlob.pop()
else: else:
print 'Module does not exist' print('Module does not exist')
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"]) printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"])
printarrayGlob.pop() printarrayGlob.pop()
@ -174,7 +174,7 @@ def waiting_refresh():
last_refresh = time.time() last_refresh = time.time()
return True return True
if __name__ == "__main__": if __name__ == "__main__":
@ -192,7 +192,7 @@ if __name__ == "__main__":
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
# REDIS # # REDIS #
@ -222,17 +222,17 @@ if __name__ == "__main__":
#while key != 'q': #while key != 'q':
# key = stdsrc.getch() # key = stdsrc.getch()
# stdscr.refresh() # stdscr.refresh()
all_queue = set() all_queue = set()
printarray1 = [] printarray1 = []
printarray2 = [] printarray2 = []
printarray3 = [] printarray3 = []
for queue, card in server.hgetall("queues").iteritems(): for queue, card in server.hgetall("queues").items():
all_queue.add(queue) all_queue.add(queue)
key = "MODULE_" + queue + "_" key = "MODULE_" + queue + "_"
keySet = "MODULE_TYPE_" + queue keySet = "MODULE_TYPE_" + queue
array_module_type = [] array_module_type = []
for moduleNum in server.smembers(keySet): for moduleNum in server.smembers(keySet):
value = server.get(key + str(moduleNum)) value = server.get(key + str(moduleNum))
if value is not None: if value is not None:
@ -240,7 +240,7 @@ if __name__ == "__main__":
if timestamp is not None and path is not None: if timestamp is not None and path is not None:
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
if int(card) > 0: if int(card) > 0:
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold: if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
log = open(log_filename, 'a') log = open(log_filename, 'a')
@ -251,15 +251,15 @@ if __name__ == "__main__":
last_kill_try = kill_retry_threshold+1 last_kill_try = kill_retry_threshold+1
if args.autokill == 1 and last_kill_try > kill_retry_threshold : if args.autokill == 1 and last_kill_try > kill_retry_threshold :
kill_module(queue, int(moduleNum)) kill_module(queue, int(moduleNum))
array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)]) array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)])
else: else:
printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)]) printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)])
array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
for e in array_module_type: for e in array_module_type:
printarray1.append(e) printarray1.append(e)
for curr_queue in module_file_array: for curr_queue in module_file_array:
if curr_queue not in all_queue: if curr_queue not in all_queue:
printarray3.append([curr_queue, "Not running"]) printarray3.append([curr_queue, "Not running"])
@ -277,16 +277,16 @@ if __name__ == "__main__":
printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"]) printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"])
else: else:
printarray3.append([curr_queue, "Stuck or idle, restarting disabled"]) printarray3.append([curr_queue, "Stuck or idle, restarting disabled"])
## FIXME To add: ## FIXME To add:
## Button KILL Process using Curses ## Button KILL Process using Curses
printarray1.sort(key=lambda x: x[0][9:], reverse=False) printarray1.sort(key=lambda x: x[0][9:], reverse=False)
printarray2.sort(key=lambda x: x[0][9:], reverse=False) printarray2.sort(key=lambda x: x[0][9:], reverse=False)
printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
printarray3.insert(0,["Queue", "State"]) printarray3.insert(0,["Queue", "State"])
os.system('clear') os.system('clear')
t1 = AsciiTable(printarray1, title="Working queues") t1 = AsciiTable(printarray1, title="Working queues")
t1.column_max_width(1) t1.column_max_width(1)
@ -304,7 +304,7 @@ if __name__ == "__main__":
temp += l + '\n' temp += l + '\n'
content[longest_col] = temp.strip() content[longest_col] = temp.strip()
t1.table_data[i] = content t1.table_data[i] = content
t2 = AsciiTable(printarray2, title="Idling queues") t2 = AsciiTable(printarray2, title="Idling queues")
t2.column_max_width(1) t2.column_max_width(1)
if not t2.ok: if not t2.ok:
@ -321,33 +321,33 @@ if __name__ == "__main__":
temp += l + '\n' temp += l + '\n'
content[longest_col] = temp.strip() content[longest_col] = temp.strip()
t2.table_data[i] = content t2.table_data[i] = content
t3 = AsciiTable(printarray3, title="Not running queues") t3 = AsciiTable(printarray3, title="Not running queues")
t3.column_max_width(1) t3.column_max_width(1)
printarray4 = [] printarray4 = []
for elem in printarrayGlob: for elem in printarrayGlob:
if elem is not None: if elem is not None:
printarray4.append(elem) printarray4.append(elem)
t4 = AsciiTable(printarray4, title="Last actions") t4 = AsciiTable(printarray4, title="Last actions")
t4.column_max_width(1) t4.column_max_width(1)
legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]] legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]]
legend = AsciiTable(legend_array, title="Legend") legend = AsciiTable(legend_array, title="Legend")
legend.column_max_width(1) legend.column_max_width(1)
print legend.table print(legend.table)
print '\n' print('\n')
print t1.table print(t1.table)
print '\n' print('\n')
print t2.table print(t2.table)
print '\n' print('\n')
print t3.table print(t3.table)
print '\n' print('\n')
print t4.table print(t4.table9)
if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
lastTime = datetime.datetime.now() lastTime = datetime.datetime.now()
cleanRedis() cleanRedis()
#time.sleep(args.refresh) #time.sleep(args.refresh)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
This module makes statistics for some modules and providers This module makes statistics for some modules and providers
@ -39,11 +39,11 @@ def compute_most_posted(server, message):
# Compute Most Posted # Compute Most Posted
date = get_date_range(0)[0] date = get_date_range(0)[0]
# check if this keyword is eligible for progression # check if this keyword is eligible for progression
keyword_total_sum = 0 keyword_total_sum = 0
curr_value = server.hget(date, module+'-'+keyword) curr_value = server.hget(date, module+'-'+keyword)
keyword_total_sum += int(curr_value) if curr_value is not None else 0 keyword_total_sum += int(curr_value) if curr_value is not None else 0
if server.zcard(redis_progression_name_set) < max_set_cardinality: if server.zcard(redis_progression_name_set) < max_set_cardinality:
server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword) server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
@ -52,17 +52,17 @@ def compute_most_posted(server, message):
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if int(member_set[0][1]) < keyword_total_sum: if int(member_set[0][1]) < keyword_total_sum:
#remove min from set and add the new one #remove min from set and add the new one
print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' print(module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
server.zrem(redis_progression_name_set, member_set[0][0]) server.zrem(redis_progression_name_set, member_set[0][0])
server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword) server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
print redis_progression_name_set print(redis_progression_name_set)
def compute_provider_info(server_trend, server_pasteName, path): def compute_provider_info(server_trend, server_pasteName, path):
redis_all_provider = 'all_provider_set' redis_all_provider = 'all_provider_set'
paste = Paste.Paste(path) paste = Paste.Paste(path)
paste_baseName = paste.p_name.split('.')[0] paste_baseName = paste.p_name.split('.')[0]
paste_size = paste._get_p_size() paste_size = paste._get_p_size()
paste_provider = paste.p_source paste_provider = paste.p_source
@ -84,7 +84,7 @@ def compute_provider_info(server_trend, server_pasteName, path):
# #
# Compute Most Posted # Compute Most Posted
# #
# Size # Size
if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil": if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil":
server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider) server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider)
@ -94,7 +94,7 @@ def compute_provider_info(server_trend, server_pasteName, path):
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if float(member_set[0][1]) < new_avg: if float(member_set[0][1]) < new_avg:
#remove min from set and add the new one #remove min from set and add the new one
print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' print('Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
server_trend.zrem(redis_sum_size_set, member_set[0][0]) server_trend.zrem(redis_sum_size_set, member_set[0][0])
server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider) server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
server_trend.zrem(redis_avg_size_name_set, member_set[0][0]) server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
@ -110,7 +110,7 @@ def compute_provider_info(server_trend, server_pasteName, path):
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if int(member_set[0][1]) < num_paste: if int(member_set[0][1]) < num_paste:
#remove min from set and add the new one #remove min from set and add the new one
print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' print('Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
server_trend.zrem(member_set[0][0]) server_trend.zrem(member_set[0][0])
server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider) server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
@ -149,7 +149,7 @@ if __name__ == '__main__':
if message is None: if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section)) publisher.debug("{} queue is empty, waiting".format(config_section))
print 'sleeping' print('sleeping')
time.sleep(20) time.sleep(20)
continue continue

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \ from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
@ -10,7 +10,7 @@ from asciimatics.event import Event
from asciimatics.event import KeyboardEvent, MouseEvent from asciimatics.event import KeyboardEvent, MouseEvent
import sys, os import sys, os
import time, datetime import time, datetime
import argparse, ConfigParser import argparse, configparser
import json import json
import redis import redis
import psutil import psutil
@ -45,7 +45,7 @@ TABLES_PADDING = {"running": [12, 23, 8, 8, 23, 10, 55, 11, 11, 12], "idle": [9,
QUEUE_STATUS = {} QUEUE_STATUS = {}
# Maintain the state of the CPU objects # Maintain the state of the CPU objects
CPU_TABLE = {} CPU_TABLE = {}
CPU_OBJECT_TABLE = {} CPU_OBJECT_TABLE = {}
# Path of the current paste for a pid # Path of the current paste for a pid
@ -137,7 +137,7 @@ class CListBox(ListBox):
# Quit if press q # Quit if press q
elif event.key_code == ord('q'): elif event.key_code == ord('q'):
Dashboard._quit() Dashboard._quit()
else: else:
# Ignore any other key press. # Ignore any other key press.
return event return event
@ -196,7 +196,7 @@ END EXTENSION
''' '''
SCENE DEFINITION SCENE DEFINITION
''' '''
class Dashboard(Frame): class Dashboard(Frame):
def __init__(self, screen): def __init__(self, screen):
@ -497,7 +497,7 @@ MANAGE MODULES AND GET INFOS
def getPid(module): def getPid(module):
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
for line in p.stdout: for line in p.stdout:
print line print(line)
splittedLine = line.split() splittedLine = line.split()
if 'python2' in splittedLine: if 'python2' in splittedLine:
return int(splittedLine[0]) return int(splittedLine[0])
@ -511,21 +511,21 @@ def clearRedisModuleInfo():
def cleanRedis(): def cleanRedis():
for k in server.keys("MODULE_TYPE_*"): for k in server.keys("MODULE_TYPE_*"):
moduleName = k[12:].split('_')[0] moduleName = (k[12:].decode('utf8')).split('_')[0]
for pid in server.smembers(k): for pid in server.smembers(k):
flag_pid_valid = False flag_pid_valid = False
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
try: try:
for line in proc.stdout: for line in proc.stdout:
splittedLine = line.split() splittedLine = line.split()
if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine: if ('python3' in splittedLine or 'python' in splittedLine) and "./"+moduleName + ".py" in splittedLine:
flag_pid_valid = True flag_pid_valid = True
if not flag_pid_valid: if not flag_pid_valid:
#print flag_pid_valid, 'cleaning', pid, 'in', k #print flag_pid_valid, 'cleaning', pid, 'in', k
server.srem(k, pid) server.srem(k, pid)
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + k], 0)) log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k).decode('utf8')], 0))
#Error due to resize, interrupted sys call #Error due to resize, interrupted sys call
except IOError as e: except IOError as e:
@ -601,12 +601,14 @@ def fetchQueueData():
printarray_running = [] printarray_running = []
printarray_idle = [] printarray_idle = []
printarray_notrunning = [] printarray_notrunning = []
for queue, card in server.hgetall("queues").iteritems(): for queue, card in iter(server.hgetall("queues").items()):
queue = queue.decode('utf8')
card = card.decode('utf8')
all_queue.add(queue) all_queue.add(queue)
key = "MODULE_" + queue + "_" key = "MODULE_" + queue + "_"
keySet = "MODULE_TYPE_" + queue keySet = "MODULE_TYPE_" + queue
array_module_type = [] array_module_type = []
for moduleNum in server.smembers(keySet): for moduleNum in server.smembers(keySet):
value = server.get(key + str(moduleNum)) value = server.get(key + str(moduleNum))
complete_paste_path = server.get(key + str(moduleNum) + "_PATH") complete_paste_path = server.get(key + str(moduleNum) + "_PATH")
@ -624,7 +626,7 @@ def fetchQueueData():
QUEUE_STATUS[moduleNum] = 1 QUEUE_STATUS[moduleNum] = 1
else: else:
QUEUE_STATUS[moduleNum] = 0 QUEUE_STATUS[moduleNum] = 0
# Queue contain elements # Queue contain elements
if int(card) > 0: if int(card) > 0:
# Queue need to be killed # Queue need to be killed
@ -636,7 +638,7 @@ def fetchQueueData():
last_kill_try = kill_retry_threshold+1 last_kill_try = kill_retry_threshold+1
if args.autokill == 1 and last_kill_try > kill_retry_threshold : if args.autokill == 1 and last_kill_try > kill_retry_threshold :
kill_module(queue, int(moduleNum)) kill_module(queue, int(moduleNum))
# Create CPU objects # Create CPU objects
try: try:
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent() cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
@ -664,9 +666,9 @@ def fetchQueueData():
mem_percent = 0 mem_percent = 0
array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable), array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable),
str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%", str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%",
"{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) ) "{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) )
else: else:
printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) ) printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
@ -674,7 +676,7 @@ def fetchQueueData():
array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes
for e in array_module_type: for e in array_module_type:
printarray_running.append(e) printarray_running.append(e)
for curr_queue in module_file_array: for curr_queue in module_file_array:
if curr_queue not in all_queue: #Module not running by default if curr_queue not in all_queue: #Module not running by default
printarray_notrunning.append( ([" <S> ", curr_queue, "Not running by default"], curr_queue) ) printarray_notrunning.append( ([" <S> ", curr_queue, "Not running by default"], curr_queue) )
@ -692,8 +694,8 @@ def fetchQueueData():
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], curr_queue) ) printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], curr_queue) )
else: else:
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], curr_queue) ) printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], curr_queue) )
printarray_running.sort(key=lambda x: x[0], reverse=False) printarray_running.sort(key=lambda x: x[0], reverse=False)
printarray_idle.sort(key=lambda x: x[0], reverse=False) printarray_idle.sort(key=lambda x: x[0], reverse=False)
printarray_notrunning.sort(key=lambda x: x[0][1], reverse=False) printarray_notrunning.sort(key=lambda x: x[0][1], reverse=False)
@ -715,6 +717,11 @@ def format_string(tab, padding_row):
text="" text=""
for ite, elem in enumerate(the_array): for ite, elem in enumerate(the_array):
try:
elem = elem.decode('utf8')
except AttributeError:
pass
if len(elem) > padding_row[ite]: if len(elem) > padding_row[ite]:
text += "*" + elem[-padding_row[ite]+6:] text += "*" + elem[-padding_row[ite]+6:]
padd_off = " "*5 padd_off = " "*5
@ -761,7 +768,7 @@ def demo(screen):
if time.time() - time_cooldown > args.refresh: if time.time() - time_cooldown > args.refresh:
cleanRedis() cleanRedis()
for key, val in fetchQueueData().iteritems(): #fetch data and put it into the tables for key, val in iter(fetchQueueData().items()): #fetch data and put it into the tables
TABLES[key] = val TABLES[key] = val
TABLES["logs"] = format_string(printarrayLog, TABLES_PADDING["logs"]) TABLES["logs"] = format_string(printarrayLog, TABLES_PADDING["logs"])
@ -790,7 +797,7 @@ if __name__ == "__main__":
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
# REDIS # # REDIS #
@ -821,7 +828,7 @@ if __name__ == "__main__":
module_file_array.add(line[:-1]) module_file_array.add(line[:-1])
cleanRedis() cleanRedis()
TABLES_TITLES["running"] = format_string([([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"],0)], TABLES_PADDING["running"])[0][0] TABLES_TITLES["running"] = format_string([([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"],0)], TABLES_PADDING["running"])[0][0]
TABLES_TITLES["idle"] = format_string([([" Action", "Queue", "PID", "Idle Time", "Last paste hash"],0)], TABLES_PADDING["idle"])[0][0] TABLES_TITLES["idle"] = format_string([([" Action", "Queue", "PID", "Idle Time", "Last paste hash"],0)], TABLES_PADDING["idle"])[0][0]
TABLES_TITLES["notRunning"] = format_string([([" Action", "Queue", "State"],0)], TABLES_PADDING["notRunning"])[0][0] TABLES_TITLES["notRunning"] = format_string([([" Action", "Queue", "State"],0)], TABLES_PADDING["notRunning"])[0][0]

View file

@ -1,10 +1,10 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import ConfigParser import configparser
import os import os
import smtplib import smtplib
from email.MIMEMultipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
""" """
@ -22,31 +22,31 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
def sendEmailNotification(recipient, term): def sendEmailNotification(recipient, term):
if not os.path.exists(configfile): if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \ raise Exception('Unable to find the configuration file. \
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv?') Or activate the virtualenv?')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
sender = cfg.get("Notifications", "sender"), sender = cfg.get("Notifications", "sender"),
sender_host = cfg.get("Notifications", "sender_host"), sender_host = cfg.get("Notifications", "sender_host"),
sender_port = cfg.getint("Notifications", "sender_port"), sender_port = cfg.getint("Notifications", "sender_port"),
sender_pw = cfg.get("Notifications", "sender_pw"), sender_pw = cfg.get("Notifications", "sender_pw"),
if isinstance(sender, tuple): if isinstance(sender, tuple):
sender = sender[0] sender = sender[0]
if isinstance(sender_host, tuple): if isinstance(sender_host, tuple):
sender_host = sender_host[0] sender_host = sender_host[0]
if isinstance(sender_port, tuple): if isinstance(sender_port, tuple):
sender_port = sender_port[0] sender_port = sender_port[0]
if isinstance(sender_pw, tuple): if isinstance(sender_pw, tuple):
sender_pw = sender_pw[0] sender_pw = sender_pw[0]
# raise an exception if any of these is None # raise an exception if any of these is None
if (sender is None or if (sender is None or
@ -62,22 +62,19 @@ def sendEmailNotification(recipient, term):
smtp_server.login(sender, sender_pw) smtp_server.login(sender, sender_pw)
else: else:
smtp_server = smtplib.SMTP(sender_host, sender_port) smtp_server = smtplib.SMTP(sender_host, sender_port)
mime_msg = MIMEMultipart() mime_msg = MIMEMultipart()
mime_msg['From'] = sender mime_msg['From'] = sender
mime_msg['To'] = recipient mime_msg['To'] = recipient
mime_msg['Subject'] = "AIL Term Alert" mime_msg['Subject'] = "AIL Term Alert"
body = "New occurrence for term: " + term body = "New occurrence for term: " + term
mime_msg.attach(MIMEText(body, 'plain')) mime_msg.attach(MIMEText(body, 'plain'))
smtp_server.sendmail(sender, recipient, mime_msg.as_string()) smtp_server.sendmail(sender, recipient, mime_msg.as_string())
smtp_server.quit() smtp_server.quit()
except Exception as e: except Exception as e:
print str(e) print(str(e))
# raise e # raise e

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_Sub_Onion Module The ZMQ_Sub_Onion Module
@ -37,7 +37,7 @@ from Helper import Process
def fetch(p, r_cache, urls, domains, path): def fetch(p, r_cache, urls, domains, path):
failed = [] failed = []
downloaded = [] downloaded = []
print len(urls), 'Urls to fetch.' print(len(urls), 'Urls to fetch.')
for url, domain in zip(urls, domains): for url, domain in zip(urls, domains):
if r_cache.exists(url) or url in failed: if r_cache.exists(url) or url in failed:
continue continue
@ -73,9 +73,9 @@ def fetch(p, r_cache, urls, domains, path):
r_cache.setbit(url, 0, 0) r_cache.setbit(url, 0, 0)
r_cache.expire(url, 3600) r_cache.expire(url, 3600)
failed.append(url) failed.append(url)
print 'Failed at downloading', url print('Failed at downloading', url)
print process.stdout.read() print(process.stdout.read())
print 'Failed:', len(failed), 'Downloaded:', len(downloaded) print('Failed:', len(failed), 'Downloaded:', len(downloaded))
if __name__ == "__main__": if __name__ == "__main__":
@ -109,7 +109,7 @@ if __name__ == "__main__":
while True: while True:
if message is not None: if message is not None:
print message print(message)
filename, score = message.split() filename, score = message.split()
# "For each new paste" # "For each new paste"
@ -152,6 +152,6 @@ if __name__ == "__main__":
prec_filename = filename prec_filename = filename
else: else:
publisher.debug("Script url is Idling 10s") publisher.debug("Script url is Idling 10s")
print 'Sleeping' print('Sleeping')
time.sleep(10) time.sleep(10)
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -30,10 +30,11 @@ def search_phone(message):
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4: if len(results) > 4:
print results print(results)
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('phone;{}'.format(message), 'alertHandler') msg = 'phone;{}'.format(message)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate #Send to duplicate
p.populate_set_out(message, 'Duplicate') p.populate_set_out(message, 'Duplicate')
stats = {} stats = {}

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
from pubsublogger import publisher from pubsublogger import publisher

View file

@ -1,9 +1,9 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import redis import redis
import argparse import argparse
import ConfigParser import configparser
import time import time
import os import os
from pubsublogger import publisher from pubsublogger import publisher
@ -14,7 +14,7 @@ def main():
"""Main Function""" """Main Function"""
# CONFIG # # CONFIG #
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read('./packages/config.cfg') cfg.read('./packages/config.cfg')
# SCRIPT PARSER # # SCRIPT PARSER #
@ -49,7 +49,7 @@ def main():
row.sort() row.sort()
table.add_rows(row, header=False) table.add_rows(row, header=False)
os.system('clear') os.system('clear')
print table.draw() print(table.draw())
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
This Module is used for term frequency. This Module is used for term frequency.
@ -76,6 +76,9 @@ if __name__ == "__main__":
dico_regex, dico_regexname_to_redis = refresh_dicos() dico_regex, dico_regexname_to_redis = refresh_dicos()
print('dico got refreshed') print('dico got refreshed')
print(dico_regex)
print(dico_regexname_to_redis)
filename = message filename = message
temp = filename.split('/') temp = filename.split('/')
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
@ -115,6 +118,6 @@ if __name__ == "__main__":
else: else:
publisher.debug("Script RegexForTermsFrequency is Idling") publisher.debug("Script RegexForTermsFrequency is Idling")
print "sleeping" print("sleeping")
time.sleep(5) time.sleep(5)
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import time import time
from packages import Paste from packages import Paste
@ -29,7 +29,7 @@ if __name__ == "__main__":
filepath = p.get_from_set() filepath = p.get_from_set()
if filepath is None: if filepath is None:
publisher.debug("Script Release is Idling 10s") publisher.debug("Script Release is Idling 10s")
print 'Sleeping' print('Sleeping')
time.sleep(10) time.sleep(10)
continue continue
@ -40,6 +40,7 @@ if __name__ == "__main__":
continue continue
to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path) to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path)
print(to_print)
if len(releases) > 30: if len(releases) > 30:
publisher.warning(to_print) publisher.warning(to_print)
else: else:

View file

@ -1,9 +1,9 @@
#!/usr/bin/python2.7 #!/usr/bin/python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import redis import redis
import argparse import argparse
import ConfigParser import configparser
from datetime import datetime from datetime import datetime
from pubsublogger import publisher from pubsublogger import publisher
@ -14,7 +14,7 @@ def main():
"""Main Function""" """Main Function"""
# CONFIG # # CONFIG #
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read('./packages/config.cfg') cfg.read('./packages/config.cfg')
# SCRIPT PARSER # # SCRIPT PARSER #

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -13,7 +13,7 @@ It test different possibility to makes some sqlInjection.
import time import time
import string import string
import urllib2 import urllib.request
import re import re
from pubsublogger import publisher from pubsublogger import publisher
from Helper import Process from Helper import Process
@ -74,8 +74,8 @@ def analyse(url, path):
if (result_path > 0) or (result_query > 0): if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path) paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1): if (result_path > 1) or (result_query > 1):
print "Detected SQL in URL: " print("Detected SQL in URL: ")
print urllib2.unquote(url) print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
publisher.warning(to_print) publisher.warning(to_print)
#Send to duplicate #Send to duplicate
@ -83,8 +83,8 @@ def analyse(url, path):
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
else: else:
print "Potential SQL injection:" print("Potential SQL injection:")
print urllib2.unquote(url) print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_path) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_path)
publisher.info(to_print) publisher.info(to_print)
@ -92,7 +92,7 @@ def analyse(url, path):
# Try to detect if the url passed might be an sql injection by appliying the regex # Try to detect if the url passed might be an sql injection by appliying the regex
# defined above on it. # defined above on it.
def is_sql_injection(url_parsed): def is_sql_injection(url_parsed):
line = urllib2.unquote(url_parsed) line = urllib.request.unquote(url_parsed)
line = string.upper(line) line = string.upper(line)
result = [] result = []
result_suspect = [] result_suspect = []
@ -114,10 +114,10 @@ def is_sql_injection(url_parsed):
result_suspect.append(line[temp_res:temp_res+len(word)]) result_suspect.append(line[temp_res:temp_res+len(word)])
if len(result)>0: if len(result)>0:
print result print(result)
return 2 return 2
elif len(result_suspect)>0: elif len(result_suspect)>0:
print result_suspect print(result_suspect)
return 1 return 1
else: else:
return 0 return 0

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
Sentiment analyser module. Sentiment analyser module.
@ -33,7 +33,7 @@ size_threshold = 250
line_max_length_threshold = 1000 line_max_length_threshold = 1000
import os import os
import ConfigParser import configparser
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile): if not os.path.exists(configfile):
@ -41,7 +41,7 @@ if not os.path.exists(configfile):
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
sentiment_lexicon_file = cfg.get("Directories", "sentiment_lexicon_file") sentiment_lexicon_file = cfg.get("Directories", "sentiment_lexicon_file")
@ -69,7 +69,7 @@ def Analyse(message, server):
combined_datetime = datetime.datetime.combine(the_date, the_time) combined_datetime = datetime.datetime.combine(the_date, the_time)
timestamp = calendar.timegm(combined_datetime.timetuple()) timestamp = calendar.timegm(combined_datetime.timetuple())
sentences = tokenize.sent_tokenize(p_content.decode('utf-8', 'ignore')) sentences = tokenize.sent_tokenize(p_content)
if len(sentences) > 0: if len(sentences) > 0:
avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0} avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0}
@ -99,7 +99,7 @@ def Analyse(message, server):
avg_score[k] = avg_score[k] / len(sentences) avg_score[k] = avg_score[k] / len(sentences)
# In redis-levelDB: {} = set, () = K-V # In redis-levelDB: {} = set, () = K-V
# {Provider_set -> provider_i} # {Provider_set -> provider_i}
# {Provider_TimestampInHour_i -> UniqID_i}_j # {Provider_TimestampInHour_i -> UniqID_i}_j
# (UniqID_i -> PasteValue_i) # (UniqID_i -> PasteValue_i)
@ -109,11 +109,11 @@ def Analyse(message, server):
provider_timestamp = provider + '_' + str(timestamp) provider_timestamp = provider + '_' + str(timestamp)
server.incr('UniqID') server.incr('UniqID')
UniqID = server.get('UniqID') UniqID = server.get('UniqID')
print provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines' print(provider_timestamp, '->', UniqID.decode('utf8'), 'dropped', num_line_removed, 'lines')
server.sadd(provider_timestamp, UniqID) server.sadd(provider_timestamp, UniqID)
server.set(UniqID, avg_score) server.set(UniqID, avg_score)
else: else:
print 'Dropped:', p_MimeType print('Dropped:', p_MimeType)
def isJSON(content): def isJSON(content):
@ -121,7 +121,7 @@ def isJSON(content):
json.loads(content) json.loads(content)
return True return True
except Exception,e: except Exception:
return False return False
import signal import signal
@ -170,4 +170,3 @@ if __name__ == '__main__':
continue continue
else: else:
signal.alarm(0) signal.alarm(0)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
This Module is used for term frequency. This Module is used for term frequency.
@ -126,6 +126,6 @@ if __name__ == "__main__":
else: else:
publisher.debug("Script RegexForTermsFrequency is Idling") publisher.debug("Script RegexForTermsFrequency is Idling")
print "sleeping" print("sleeping")
time.sleep(5) time.sleep(5)
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_Feed_Q Module The ZMQ_Feed_Q Module
@ -21,7 +21,7 @@ Requirements
""" """
import redis import redis
import ConfigParser import configparser
import os import os
configfile = os.path.join(os.environ['AIL_BIN'], './packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], './packages/config.cfg')
@ -31,7 +31,7 @@ def main():
"""Main Function""" """Main Function"""
# CONFIG # # CONFIG #
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
# REDIS # REDIS

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import time import time
from packages import Paste from packages import Paste
@ -26,10 +26,10 @@ if __name__ == "__main__":
adr = "0x[a-f0-9]{2}" adr = "0x[a-f0-9]{2}"
#asm = "\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\.. #asm = "\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\..
languages = [shell, c, php, bash, python, javascript, bash, ruby, adr] languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
regex = '|'.join(languages) regex = '|'.join(languages)
print regex print(regex)
while True: while True:
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The Tokenize Module The Tokenize Module
@ -50,7 +50,7 @@ if __name__ == "__main__":
while True: while True:
message = p.get_from_set() message = p.get_from_set()
print message print(message)
if message is not None: if message is not None:
paste = Paste.Paste(message) paste = Paste.Paste(message)
signal.alarm(5) signal.alarm(5)
@ -67,4 +67,4 @@ if __name__ == "__main__":
else: else:
publisher.debug("Tokeniser is idling 10s") publisher.debug("Tokeniser is idling 10s")
time.sleep(10) time.sleep(10)
print "sleepin" print("Sleeping")

View file

@ -1,8 +1,8 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import ConfigParser import configparser
from ConfigParser import ConfigParser as cfgP from configparser import ConfigParser as cfgP
import os import os
from collections import OrderedDict from collections import OrderedDict
import sys import sys
@ -20,14 +20,14 @@ def main():
Or activate the virtualenv.') Or activate the virtualenv.')
configfileSample = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample') configfileSample = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
cfgSample = ConfigParser.ConfigParser() cfgSample = configparser.ConfigParser()
cfgSample.read(configfileSample) cfgSample.read(configfileSample)
sections = cfgP.sections(cfg) sections = cfgP.sections(cfg)
sectionsSample = cfgP.sections(cfgSample) sectionsSample = cfgP.sections(cfgSample)
missingSection = [] missingSection = []
dicoMissingSection = {} dicoMissingSection = {}
missingItem = [] missingItem = []
@ -89,7 +89,7 @@ def main():
''' Return a new dico with the section ordered as the old configuration with the updated one added ''' ''' Return a new dico with the section ordered as the old configuration with the updated one added '''
def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection): def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection):
new_dico = OrderedDict() new_dico = OrderedDict()
positions = {} positions = {}
for pos_i, sec in enumerate(sample_dico): for pos_i, sec in enumerate(sample_dico):
if sec in missingSection: if sec in missingSection:
@ -109,4 +109,3 @@ if __name__ == "__main__":
sys.exit() sys.exit()
else: else:
sys.exit(1) sys.exit(1)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -100,7 +100,7 @@ if __name__ == "__main__":
publisher.debug('{} Published'.format(url)) publisher.debug('{} Published'.format(url))
if f1 == "onion": if f1 == "onion":
print domain print(domain)
hostl = unicode(avoidNone(subdomain)+avoidNone(domain)) hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
try: try:
@ -121,16 +121,16 @@ if __name__ == "__main__":
# EU is not an official ISO 3166 code (but used by RIPE # EU is not an official ISO 3166 code (but used by RIPE
# IP allocation) # IP allocation)
if cc is not None and cc != "EU": if cc is not None and cc != "EU":
print hostl, asn, cc, \ print(hostl, asn, cc, \
pycountry.countries.get(alpha_2=cc).name pycountry.countries.get(alpha_2=cc).name)
if cc == cc_critical: if cc == cc_critical:
to_print = 'Url;{};{};{};Detected {} {}'.format( to_print = 'Url;{};{};{};Detected {} {}'.format(
PST.p_source, PST.p_date, PST.p_name, PST.p_source, PST.p_date, PST.p_name,
hostl, cc) hostl, cc)
#publisher.warning(to_print) #publisher.warning(to_print)
print to_print print(to_print)
else: else:
print hostl, asn, cc print(hostl, asn, cc)
A_values = lib_refine.checking_A_record(r_serv2, A_values = lib_refine.checking_A_record(r_serv2,
domains_list) domains_list)
@ -146,7 +146,7 @@ if __name__ == "__main__":
else: else:
publisher.debug("Script url is Idling 10s") publisher.debug("Script url is Idling 10s")
print 'Sleeping' print('Sleeping')
time.sleep(10) time.sleep(10)
message = p.get_from_set() message = p.get_from_set()

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -33,7 +33,7 @@ def analyse(server, field_name, date, url_parsed):
if field_name == "domain": #save domain in a set for the monthly plot if field_name == "domain": #save domain in a set for the monthly plot
domain_set_name = "domain_set_" + date[0:6] domain_set_name = "domain_set_" + date[0:6]
server.sadd(domain_set_name, field) server.sadd(domain_set_name, field)
print "added in " + domain_set_name +": "+ field print("added in " + domain_set_name +": "+ field)
def get_date_range(num_day): def get_date_range(num_day):
curr_date = datetime.date.today() curr_date = datetime.date.today()
@ -122,7 +122,7 @@ if __name__ == '__main__':
p.config.get("Directories", "protocolstrending_csv")) p.config.get("Directories", "protocolstrending_csv"))
protocolsfile_path = os.path.join(os.environ['AIL_HOME'], protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile")) p.config.get("Directories", "protocolsfile"))
csv_path_tld = os.path.join(os.environ['AIL_HOME'], csv_path_tld = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldstrending_csv")) p.config.get("Directories", "tldstrending_csv"))
tldsfile_path = os.path.join(os.environ['AIL_HOME'], tldsfile_path = os.path.join(os.environ['AIL_HOME'],
@ -145,24 +145,25 @@ if __name__ == '__main__':
year = today.year year = today.year
month = today.month month = today.month
print 'Building protocol graph' print('Building protocol graph')
lib_words.create_curve_with_word_file(r_serv_trend, csv_path_proto, lib_words.create_curve_with_word_file(r_serv_trend, csv_path_proto,
protocolsfile_path, year, protocolsfile_path, year,
month) month)
print 'Building tld graph' print('Building tld graph')
lib_words.create_curve_with_word_file(r_serv_trend, csv_path_tld, lib_words.create_curve_with_word_file(r_serv_trend, csv_path_tld,
tldsfile_path, year, tldsfile_path, year,
month) month)
print 'Building domain graph' print('Building domain graph')
lib_words.create_curve_from_redis_set(r_serv_trend, csv_path_domain, lib_words.create_curve_from_redis_set(r_serv_trend, csv_path_domain,
"domain", year, "domain", year,
month) month)
print 'end building' print('end building')
publisher.debug("{} queue is empty, waiting".format(config_section)) publisher.debug("{} queue is empty, waiting".format(config_section))
print 'sleeping' print('sleeping')
time.sleep(5*60) time.sleep(5*60)
continue continue
@ -172,10 +173,14 @@ if __name__ == '__main__':
url, date, path = message.split() url, date, path = message.split()
faup.decode(url) faup.decode(url)
url_parsed = faup.get() url_parsed = faup.get()
analyse(r_serv_trend, 'scheme', date, url_parsed) #Scheme analysis # Scheme analysis
analyse(r_serv_trend, 'tld', date, url_parsed) #Tld analysis analyse(r_serv_trend, 'scheme', date, url_parsed)
analyse(r_serv_trend, 'domain', date, url_parsed) #Domain analysis # Tld analysis
analyse(r_serv_trend, 'tld', date, url_parsed)
# Domain analysis
analyse(r_serv_trend, 'domain', date, url_parsed)
compute_progression(r_serv_trend, 'scheme', num_day_to_look, url_parsed) compute_progression(r_serv_trend, 'scheme', num_day_to_look, url_parsed)
compute_progression(r_serv_trend, 'tld', num_day_to_look, url_parsed) compute_progression(r_serv_trend, 'tld', num_day_to_look, url_parsed)
compute_progression(r_serv_trend, 'domain', num_day_to_look, url_parsed) compute_progression(r_serv_trend, 'domain', num_day_to_look, url_parsed)

View file

@ -62,8 +62,9 @@ if __name__ == "__main__":
while True: while True:
message = p.get_from_set() message = p.get_from_set()
if message is not None: if message is not None:
message = message.decode('utf8') #decode because of pyhton3 #decode because of pyhton3
module_name, p_path = message.split(';') module_name, p_path = message.split(';')
print("new alert : {}".format(module_name))
#PST = Paste.Paste(p_path) #PST = Paste.Paste(p_path)
else: else:
publisher.debug("Script Attribute is idling 10s") publisher.debug("Script Attribute is idling 10s")

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -27,10 +27,9 @@ if __name__ == "__main__":
config_section = ['Curve'] config_section = ['Curve']
for queue in config_section: for queue in config_section:
print 'dropping: ' + queue print('dropping: ' + queue)
p = Process(queue) p = Process(queue)
while True: while True:
message = p.get_from_set() message = p.get_from_set()
if message is None: if message is None:
break break

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# This file is part of AIL framework - Analysis Information Leak framework # This file is part of AIL framework - Analysis Information Leak framework
@ -25,7 +25,7 @@ import time
import redis import redis
import base64 import base64
import os import os
import ConfigParser import configparser
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile): if not os.path.exists(configfile):
@ -33,7 +33,7 @@ if not os.path.exists(configfile):
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
if cfg.has_option("ZMQ_Global", "bind"): if cfg.has_option("ZMQ_Global", "bind"):
@ -59,6 +59,7 @@ while True:
time.sleep(base_sleeptime + sleep_inc) time.sleep(base_sleeptime + sleep_inc)
topic = 101 topic = 101
paste = r.lpop("pastes") paste = r.lpop("pastes")
print(paste)
if paste is None: if paste is None:
continue continue
socket.send("%d %s" % (topic, paste)) socket.send("%d %s" % (topic, paste))

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# This file is part of AIL framework - Analysis Information Leak framework # This file is part of AIL framework - Analysis Information Leak framework
@ -19,14 +19,15 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
# 101 Name of the pastes only # 101 Name of the pastes only
# 102 Full pastes in raw base64(gz) # 102 Full pastes in raw base64(gz)
topicfilter = "102" topicfilter = b"102"
socket.setsockopt(zmq.SUBSCRIBE, topicfilter) socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
print('b0')
while True: while True:
message = socket.recv() message = socket.recv()
print('b1')
print (message) print (message)
if topicfilter == "102": if topicfilter == b"102":
topic, paste, messagedata = message.split() topic, paste, messagedata = message.split()
print paste, messagedata print(paste, messagedata)
else: else:
print (message) print (message)

View file

@ -1,9 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import zmq import zmq
import base64 import base64
import StringIO from io import StringIO
import gzip import gzip
import argparse import argparse
import os import os
@ -31,8 +31,7 @@ import mimetypes
' '
''' '''
import StringIO
import gzip
def is_hierachy_valid(path): def is_hierachy_valid(path):
var = path.split('/') var = path.split('/')
try: try:
@ -72,7 +71,12 @@ if __name__ == "__main__":
wanted_path = wanted_path.split('/') wanted_path = wanted_path.split('/')
wanted_path = '/'.join(wanted_path[-(4+args.hierarchy):]) wanted_path = '/'.join(wanted_path[-(4+args.hierarchy):])
messagedata = open(complete_path).read() with gzip.open(complete_path, 'rb') as f:
messagedata = f.read()
#print(type(complete_path))
#file = open(complete_path)
#messagedata = file.read()
#if paste do not have a 'date hierarchy' ignore it #if paste do not have a 'date hierarchy' ignore it
if not is_hierachy_valid(complete_path): if not is_hierachy_valid(complete_path):
@ -90,5 +94,8 @@ if __name__ == "__main__":
print(args.name+'>'+wanted_path) print(args.name+'>'+wanted_path)
path_to_send = args.name + '>' + wanted_path path_to_send = args.name + '>' + wanted_path
socket.send('{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata))) #s = b'{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata))
# use bytes object
s = b' '.join( [ args.channel.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] )
socket.send(s)
time.sleep(args.seconds) time.sleep(args.seconds)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# This file is part of AIL framework - Analysis Information Leak framework # This file is part of AIL framework - Analysis Information Leak framework
@ -10,7 +10,7 @@
# #
# Copyright (c) 2014 Alexandre Dulaunoy - a@foo.be # Copyright (c) 2014 Alexandre Dulaunoy - a@foo.be
import ConfigParser import configparser
import argparse import argparse
import gzip import gzip
import os import os
@ -23,7 +23,7 @@ def readdoc(path=None):
return f.read() return f.read()
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
# Indexer configuration - index dir and schema setup # Indexer configuration - index dir and schema setup
@ -51,7 +51,7 @@ ix = index.open_dir(indexpath)
from whoosh.qparser import QueryParser from whoosh.qparser import QueryParser
if args.n: if args.n:
print ix.doc_count_all() print(ix.doc_count_all())
exit(0) exit(0)
if args.l: if args.l:

View file

@ -1,8 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import ConfigParser import configparser
import os import os
import subprocess import subprocess
import time import time
@ -23,21 +23,21 @@ if __name__ == '__main__':
raise Exception('Unable to find the configuration file. \ raise Exception('Unable to find the configuration file. \
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
config = ConfigParser.ConfigParser() config = configparser.ConfigParser()
config.read(configfile) config.read(configfile)
modules = config.sections() modules = config.sections()
pids = {} pids = {}
for module in modules: for module in modules:
pin = subprocess.Popen(["python", './QueueIn.py', '-c', module]) pin = subprocess.Popen(["python3", './QueueIn.py', '-c', module])
pout = subprocess.Popen(["python", './QueueOut.py', '-c', module]) pout = subprocess.Popen(["python3", './QueueOut.py', '-c', module])
pids[module] = (pin, pout) pids[module] = (pin, pout)
is_running = True is_running = True
try: try:
while is_running: while is_running:
time.sleep(5) time.sleep(5)
is_running = False is_running = False
for module, p in pids.iteritems(): for module, p in pids.items():
pin, pout = p pin, pout = p
if pin is None: if pin is None:
# already dead # already dead
@ -57,7 +57,7 @@ if __name__ == '__main__':
is_running = True is_running = True
pids[module] = (pin, pout) pids[module] = (pin, pout)
except KeyboardInterrupt: except KeyboardInterrupt:
for module, p in pids.iteritems(): for module, p in pids.items():
pin, pout = p pin, pout = p
if pin is not None: if pin is not None:
pin.kill() pin.kill()

View file

@ -1,4 +1,4 @@
#!/usr/bin/python2.7 #!/usr/bin/python3.5
""" """
The ``Paste Class`` The ``Paste Class``
@ -24,15 +24,8 @@ import operator
import string import string
import re import re
import json import json
try: # dirty to support python3 import configparser
import ConfigParser from io import StringIO
except:
import configparser
ConfigParser = configparser
try: # dirty to support python3
import cStringIO
except:
from io import StringIO as cStringIO
import sys import sys
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
from Date import Date from Date import Date
@ -71,7 +64,7 @@ class Paste(object):
Did you set environment variables? \ Did you set environment variables? \
Or activate the virtualenv.') Or activate the virtualenv.')
cfg = ConfigParser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
self.cache = redis.StrictRedis( self.cache = redis.StrictRedis(
host=cfg.get("Redis_Queues", "host"), host=cfg.get("Redis_Queues", "host"),
@ -85,11 +78,15 @@ class Paste(object):
self.p_path = p_path self.p_path = p_path
self.p_name = os.path.basename(self.p_path) self.p_name = os.path.basename(self.p_path)
self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2) self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2)
self.p_mime = magic.from_buffer("test", mime=True)
self.p_mime = magic.from_buffer(self.get_p_content(), mime=True) self.p_mime = magic.from_buffer(self.get_p_content(), mime=True)
# Assuming that the paste will alway be in a day folder which is itself # Assuming that the paste will alway be in a day folder which is itself
# in a month folder which is itself in a year folder. # in a month folder which is itself in a year folder.
# /year/month/day/paste.gz # /year/month/day/paste.gz
# TODO use bytes ?
var = self.p_path.split('/') var = self.p_path.split('/')
self.p_date = Date(var[-4], var[-3], var[-2]) self.p_date = Date(var[-4], var[-3], var[-2])
self.p_source = var[-5] self.p_source = var[-5]
@ -117,17 +114,25 @@ class Paste(object):
paste = self.cache.get(self.p_path) paste = self.cache.get(self.p_path)
if paste is None: if paste is None:
try: try:
#print('----------------------------------------------------------------')
#print(self.p_name)
#print('----------------------------------------------------------------')
with gzip.open(self.p_path, 'rb') as f: with gzip.open(self.p_path, 'rb') as f:
paste = f.read() paste = f.read()
self.cache.set(self.p_path, paste) self.cache.set(self.p_path, paste)
self.cache.expire(self.p_path, 300) self.cache.expire(self.p_path, 300)
except: except:
return '' paste = b''
pass
return paste return paste.decode('utf8')
def get_p_content_as_file(self): def get_p_content_as_file(self):
return cStringIO.StringIO(self.get_p_content()) try:
message = StringIO( (self.get_p_content()).decode('utf8') )
except AttributeError:
message = StringIO( (self.get_p_content()) )
return message
def get_p_content_with_removed_lines(self, threshold): def get_p_content_with_removed_lines(self, threshold):
num_line_removed = 0 num_line_removed = 0
@ -137,6 +142,7 @@ class Paste(object):
line_id = 0 line_id = 0
for line_id, line in enumerate(f): for line_id, line in enumerate(f):
length = len(line) length = len(line)
if length < line_length_threshold: if length < line_length_threshold:
string_content += line string_content += line
else: else:
@ -202,8 +208,8 @@ class Paste(object):
.. seealso:: _set_p_hash_kind("md5") .. seealso:: _set_p_hash_kind("md5")
""" """
for hash_name, the_hash in self.p_hash_kind.iteritems(): for hash_name, the_hash in self.p_hash_kind.items():
self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content()) self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content().encode('utf8'))
return self.p_hash return self.p_hash
def _get_p_language(self): def _get_p_language(self):
@ -271,7 +277,7 @@ class Paste(object):
return True, var return True, var
else: else:
return False, var return False, var
def _get_p_duplicate(self): def _get_p_duplicate(self):
self.p_duplicate = self.store.hget(self.p_path, "p_duplicate") self.p_duplicate = self.store.hget(self.p_path, "p_duplicate")
return self.p_duplicate if self.p_duplicate is not None else '[]' return self.p_duplicate if self.p_duplicate is not None else '[]'
@ -342,7 +348,7 @@ class Paste(object):
tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+', tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
gaps=True, discard_empty=True) gaps=True, discard_empty=True)
blob = TextBlob(clean(self.get_p_content()), tokenizer=tokenizer) blob = TextBlob(clean( (self.get_p_content()) ), tokenizer=tokenizer)
for word in blob.tokens: for word in blob.tokens:
if word in words.keys(): if word in words.keys():
@ -351,7 +357,7 @@ class Paste(object):
num = 0 num = 0
words[word] = num + 1 words[word] = num + 1
if sort: if sort:
var = sorted(words.iteritems(), key=operator.itemgetter(1), reverse=True) var = sorted(words.items(), key=operator.itemgetter(1), reverse=True)
else: else:
var = words var = words

View file

@ -76,7 +76,7 @@ def checking_MX_record(r_serv, adress_set):
r_serv.setex(MXdomain[1:], 1, timedelta(days=1)) r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
except Exception as e: except Exception as e:
print e print(e)
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score)) publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
return (num, WalidMX) return (num, WalidMX)
@ -125,7 +125,7 @@ def checking_A_record(r_serv, domains_set):
publisher.debug('The Label is too long') publisher.debug('The Label is too long')
except Exception as e: except Exception as e:
print e print(e)
publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score)) publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
return (num, WalidA) return (num, WalidA)

View file

@ -82,16 +82,16 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
""" """
threshold = 50 threshold = 50
first_day = date(year, month, 01) first_day = date(year, month, 1)
last_day = date(year, month, calendar.monthrange(year, month)[1]) last_day = date(year, month, calendar.monthrange(year, month)[1])
words = [] words = []
with open(feederfilename, 'rb') as f: with open(feederfilename, 'r') as f:
# words of the files # words of the files
words = sorted([word.strip() for word in f if word.strip()[0:2]!='//' and word.strip()!='' ]) words = sorted([word.strip() for word in f if word.strip()[0:2]!='//' and word.strip()!='' ])
headers = ['Date'] + words headers = ['Date'] + words
with open(csvfilename+'.csv', 'wb') as f: with open(csvfilename+'.csv', 'w') as f:
writer = csv.writer(f) writer = csv.writer(f)
writer.writerow(headers) writer.writerow(headers)
@ -103,11 +103,14 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
# from the 1srt day to the last of the list # from the 1srt day to the last of the list
for word in words: for word in words:
value = r_serv.hget(word, curdate) value = r_serv.hget(word, curdate)
if value is None: if value is None:
row.append(0) row.append(0)
else: else:
# if the word have a value for the day # if the word have a value for the day
# FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold # FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold
value = r_serv.hget(word, curdate)
value = int(value.decode('utf8'))
if value >= threshold: if value >= threshold:
row.append(value) row.append(value)
writer.writerow(row) writer.writerow(row)
@ -127,14 +130,14 @@ def create_curve_from_redis_set(server, csvfilename, set_to_plot, year, month):
""" """
first_day = date(year, month, 01) first_day = date(year, month, 1)
last_day = date(year, month, calendar.monthrange(year, month)[1]) last_day = date(year, month, calendar.monthrange(year, month)[1])
redis_set_name = set_to_plot + "_set_" + str(year) + str(month).zfill(2) redis_set_name = set_to_plot + "_set_" + str(year) + str(month).zfill(2)
words = list(server.smembers(redis_set_name)) words = list(server.smembers(redis_set_name))
headers = ['Date'] + words headers = ['Date'] + words
with open(csvfilename+'.csv', 'wb') as f: with open(csvfilename+'.csv', 'w') as f:
writer = csv.writer(f) writer = csv.writer(f)
writer.writerow(headers) writer.writerow(headers)

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
''' '''
@ -48,7 +48,7 @@ if __name__ == '__main__':
message = p.get_from_set() message = p.get_from_set()
if message is None: if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section)) publisher.debug("{} queue is empty, waiting".format(config_section))
print "queue empty" print("queue empty")
time.sleep(1) time.sleep(1)
continue continue

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
Template for new modules Template for new modules

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import socks import socks
@ -63,5 +63,5 @@ if __name__ == "__main__":
t, path = tempfile.mkstemp() t, path = tempfile.mkstemp()
with open(path, 'w') as f: with open(path, 'w') as f:
f.write(to_write) f.write(to_write)
print path print(path)
exit(0) exit(0)