mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
decode with redis connection
This commit is contained in:
parent
7ae0f3c837
commit
96a3df42bc
72 changed files with 222 additions and 448 deletions
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -65,12 +65,6 @@ if __name__ == "__main__":
|
|||
|
||||
publisher.info("ApiKey started")
|
||||
|
||||
# REDIS #
|
||||
r_serv2 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Cache", "host"),
|
||||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
|
||||
message = p.get_from_set()
|
||||
|
||||
# TODO improve REGEX
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Base64 module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Bitcoin Module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The ZMQ_PubSub_Categ Module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -55,7 +55,8 @@ if __name__ == "__main__":
|
|||
server_cred = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
||||
db=p.config.get("Redis_Level_DB_TermCred", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
criticalNumberToAlert = p.config.getint("Credential", "criticalNumberToAlert")
|
||||
minTopPassList = p.config.getint("Credential", "minTopPassList")
|
||||
|
@ -68,8 +69,8 @@ if __name__ == "__main__":
|
|||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("Script Credential is Idling 10s")
|
||||
print('sleeping 10s')
|
||||
time.sleep(1)
|
||||
#print('sleeping 10s')
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
filepath, count = message.split(' ')
|
||||
|
@ -109,7 +110,7 @@ if __name__ == "__main__":
|
|||
site_occurence = re.findall(regex_site_for_stats, content)
|
||||
for site in site_occurence:
|
||||
site_domain = site[1:-1]
|
||||
if site_domain.encode('utf8') in creds_sites.keys():
|
||||
if site_domain in creds_sites.keys():
|
||||
creds_sites[site_domain] += 1
|
||||
else:
|
||||
creds_sites[site_domain] = 1
|
||||
|
@ -123,10 +124,6 @@ if __name__ == "__main__":
|
|||
creds_sites[domain] = 1
|
||||
|
||||
for site, num in creds_sites.items(): # Send for each different site to moduleStats
|
||||
try:
|
||||
site = site.decode('utf8')
|
||||
except:
|
||||
pass
|
||||
|
||||
mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
|
||||
print(mssg)
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
12
bin/Curve.py
12
bin/Curve.py
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
|
||||
|
@ -49,7 +49,7 @@ top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
|
|||
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
|
||||
|
||||
def check_if_tracked_term(term, path):
|
||||
if term.encode('utf8') in server_term.smembers(TrackedTermsSet_Name):
|
||||
if term in server_term.smembers(TrackedTermsSet_Name):
|
||||
#add_paste to tracked_word_set
|
||||
set_name = "tracked_" + term
|
||||
server_term.sadd(set_name, path)
|
||||
|
@ -84,12 +84,14 @@ if __name__ == "__main__":
|
|||
r_serv1 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_Curve", "host"),
|
||||
port=p.config.get("Redis_Level_DB_Curve", "port"),
|
||||
db=p.config.get("Redis_Level_DB_Curve", "db"))
|
||||
db=p.config.get("Redis_Level_DB_Curve", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
server_term = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script Curve started")
|
||||
|
@ -132,7 +134,7 @@ if __name__ == "__main__":
|
|||
curr_word_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), low_word, int(1)))
|
||||
|
||||
# Add in set only if term is not in the blacklist
|
||||
if low_word.encode('utf8') not in server_term.smembers(BlackListTermsSet_Name):
|
||||
if low_word not in server_term.smembers(BlackListTermsSet_Name):
|
||||
#consider the num of occurence of this term
|
||||
server_term.zincrby(curr_set, low_word, float(score))
|
||||
#1 term per paste
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
|
||||
|
@ -139,7 +139,8 @@ if __name__ == '__main__':
|
|||
r_temp = redis.StrictRedis(
|
||||
host=cfg.get('RedisPubSub', 'host'),
|
||||
port=cfg.getint('RedisPubSub', 'port'),
|
||||
db=cfg.getint('RedisPubSub', 'db'))
|
||||
db=cfg.getint('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
|
||||
value = str(timestamp) + ", " + "-"
|
||||
|
@ -149,7 +150,8 @@ if __name__ == '__main__':
|
|||
server_term = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_TermFreq", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
publisher.info("Script Curve_manage_top_set started")
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The CVE Module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import argparse
|
||||
|
@ -36,7 +36,8 @@ def main():
|
|||
|
||||
r_serv = redis.StrictRedis(host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -53,7 +53,8 @@ if __name__ == "__main__":
|
|||
for month in range(0, 13):
|
||||
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"), port=year,
|
||||
db=month)
|
||||
db=month,
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
@ -102,7 +103,7 @@ if __name__ == "__main__":
|
|||
yearly_index = str(date_today.year)+'00'
|
||||
r_serv0 = dico_redis[yearly_index]
|
||||
r_serv0.incr("current_index")
|
||||
index = (r_serv0.get("current_index")).decode('utf8') + str(PST.p_date)
|
||||
index = (r_serv0.get("current_index")) + str(PST.p_date)
|
||||
|
||||
# Open selected dico range
|
||||
opened_dico = []
|
||||
|
@ -116,7 +117,7 @@ if __name__ == "__main__":
|
|||
for curr_dico_name, curr_dico_redis in opened_dico:
|
||||
for hash_type, paste_hash in paste_hashes.items():
|
||||
for dico_hash in curr_dico_redis.smembers('HASHS_'+hash_type):
|
||||
dico_hash = dico_hash.decode('utf8')
|
||||
dico_hash = dico_hash
|
||||
|
||||
try:
|
||||
if hash_type == 'ssdeep':
|
||||
|
@ -134,11 +135,11 @@ if __name__ == "__main__":
|
|||
|
||||
# index of paste
|
||||
index_current = r_serv_dico.get(dico_hash)
|
||||
index_current = index_current.decode('utf8')
|
||||
index_current = index_current
|
||||
paste_path = r_serv_dico.get(index_current)
|
||||
paste_path = paste_path.decode('utf8')
|
||||
paste_path = paste_path
|
||||
paste_date = r_serv_dico.get(index_current+'_date')
|
||||
paste_date = paste_date.decode('utf8')
|
||||
paste_date = paste_date
|
||||
paste_date = paste_date if paste_date != None else "No date available"
|
||||
if paste_path != None:
|
||||
if paste_path != PST.p_path:
|
||||
|
|
|
@ -1,166 +0,0 @@
|
|||
#!/usr/bin/env python3.5
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Duplicate module
|
||||
====================
|
||||
|
||||
This huge module is, in short term, checking duplicates.
|
||||
|
||||
Requirements:
|
||||
-------------
|
||||
|
||||
|
||||
"""
|
||||
import redis
|
||||
import os
|
||||
import time
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from pybloomfilter import BloomFilter
|
||||
|
||||
from Helper import Process
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'Duplicates'
|
||||
|
||||
p = Process(config_section)
|
||||
|
||||
# REDIS #
|
||||
# DB OBJECT & HASHS ( DISK )
|
||||
# FIXME increase flexibility
|
||||
dico_redis = {}
|
||||
for year in range(2013, 2017):
|
||||
for month in range(0, 16):
|
||||
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"), port=year,
|
||||
db=month)
|
||||
#print("dup: "+str(year)+str(month).zfill(2)+"\n")
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
||||
set_limit = 100
|
||||
bloompath = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "bloomfilters"))
|
||||
|
||||
bloop_path_set = set()
|
||||
while True:
|
||||
try:
|
||||
super_dico = {}
|
||||
hash_dico = {}
|
||||
dupl = []
|
||||
nb_hash_current = 0
|
||||
|
||||
x = time.time()
|
||||
|
||||
message = p.get_from_set()
|
||||
if message is not None:
|
||||
path = message
|
||||
PST = Paste.Paste(path)
|
||||
else:
|
||||
publisher.debug("Script Attribute is idling 10s")
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
PST._set_p_hash_kind("md5")
|
||||
|
||||
# Assignate the correct redis connexion
|
||||
r_serv1 = dico_redis[PST.p_date.year + PST.p_date.month]
|
||||
|
||||
# Creating the bloom filter name: bloomyyyymm
|
||||
filebloompath = os.path.join(bloompath, 'bloom' + PST.p_date.year +
|
||||
PST.p_date.month)
|
||||
if os.path.exists(filebloompath):
|
||||
bloom = BloomFilter.open(filebloompath)
|
||||
bloop_path_set.add(filebloompath)
|
||||
else:
|
||||
bloom = BloomFilter(100000000, 0.01, filebloompath)
|
||||
bloop_path_set.add(filebloompath)
|
||||
|
||||
# UNIQUE INDEX HASHS TABLE
|
||||
r_serv0 = dico_redis["201600"]
|
||||
r_serv0.incr("current_index")
|
||||
index = r_serv0.get("current_index")+str(PST.p_date)
|
||||
# HASHTABLES PER MONTH (because of r_serv1 changing db)
|
||||
r_serv1.set(index, PST.p_path)
|
||||
r_serv1.sadd("INDEX", index)
|
||||
|
||||
# For each bloom filter
|
||||
opened_bloom = []
|
||||
for bloo in bloop_path_set:
|
||||
# Opening blooms
|
||||
opened_bloom.append(BloomFilter.open(bloo))
|
||||
# For each hash of the paste
|
||||
for line_hash in PST._get_hash_lines(min=5, start=1, jump=0):
|
||||
nb_hash_current += 1
|
||||
|
||||
# Adding the hash in Redis & limiting the set
|
||||
if r_serv1.scard(line_hash) <= set_limit:
|
||||
r_serv1.sadd(line_hash, index)
|
||||
r_serv1.sadd("HASHS", line_hash)
|
||||
# Adding the hash in the bloom of the month
|
||||
bloom.add(line_hash)
|
||||
# Go throught the Database of the bloom filter (of the month)
|
||||
for bloo in opened_bloom:
|
||||
if line_hash in bloo:
|
||||
db = bloo.name[-6:]
|
||||
# Go throught the Database of the bloom filter (month)
|
||||
r_serv_bloom = dico_redis[db]
|
||||
|
||||
# set of index paste: set([1,2,4,65])
|
||||
hash_current = r_serv_bloom.smembers(line_hash)
|
||||
# removing itself from the list
|
||||
hash_current = hash_current - set([index])
|
||||
|
||||
# if the hash is present at least in 1 files
|
||||
# (already processed)
|
||||
if len(hash_current) != 0:
|
||||
hash_dico[line_hash] = hash_current
|
||||
|
||||
# if there is data in this dictionnary
|
||||
if len(hash_dico) != 0:
|
||||
super_dico[index] = hash_dico
|
||||
|
||||
###########################################################################
|
||||
|
||||
# if there is data in this dictionnary
|
||||
if len(super_dico) != 0:
|
||||
# current = current paste, phash_dico = {hash: set, ...}
|
||||
occur_dico = {}
|
||||
for current, phash_dico in super_dico.items():
|
||||
# phash = hash, pset = set([ pastes ...])
|
||||
for phash, pset in hash_dico.items():
|
||||
|
||||
for p_fname in pset:
|
||||
occur_dico.setdefault(p_fname, 0)
|
||||
# Count how much hash is similar per file occuring
|
||||
# in the dictionnary
|
||||
if occur_dico[p_fname] >= 0:
|
||||
occur_dico[p_fname] = occur_dico[p_fname] + 1
|
||||
|
||||
for paste, count in occur_dico.items():
|
||||
percentage = round((count/float(nb_hash_current))*100, 2)
|
||||
if percentage >= 50:
|
||||
dupl.append((paste, percentage))
|
||||
else:
|
||||
print('percentage: ' + str(percentage))
|
||||
|
||||
# Creating the object attribute and save it.
|
||||
to_print = 'Duplicate;{};{};{};'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name)
|
||||
if dupl != []:
|
||||
PST.__setattr__("p_duplicate", dupl)
|
||||
PST.save_attribute_redis("p_duplicate", dupl)
|
||||
publisher.info('{}Detected {}'.format(to_print, len(dupl)))
|
||||
print('{}Detected {}'.format(to_print, len(dupl)))
|
||||
|
||||
y = time.time()
|
||||
|
||||
publisher.debug('{}Processed in {} sec'.format(to_print, y-x))
|
||||
except IOError:
|
||||
print("CRC Checksum Failed on :", PST.p_path)
|
||||
publisher.error('{}CRC Checksum Failed'.format(to_print))
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The ZMQ_Feed_Q Module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Queue helper module
|
||||
|
@ -45,7 +45,8 @@ class PubSub(object):
|
|||
r = redis.StrictRedis(
|
||||
host=self.config.get('RedisPubSub', 'host'),
|
||||
port=self.config.get('RedisPubSub', 'port'),
|
||||
db=self.config.get('RedisPubSub', 'db'))
|
||||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
self.subscribers = r.pubsub(ignore_subscribe_messages=True)
|
||||
self.subscribers.psubscribe(channel)
|
||||
elif conn_name.startswith('ZMQ'):
|
||||
|
@ -69,7 +70,8 @@ class PubSub(object):
|
|||
if conn_name.startswith('Redis'):
|
||||
r = redis.StrictRedis(host=self.config.get('RedisPubSub', 'host'),
|
||||
port=self.config.get('RedisPubSub', 'port'),
|
||||
db=self.config.get('RedisPubSub', 'db'))
|
||||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
self.publishers['Redis'].append((r, channel))
|
||||
elif conn_name.startswith('ZMQ'):
|
||||
context = zmq.Context()
|
||||
|
@ -99,8 +101,7 @@ class PubSub(object):
|
|||
for sub in self.subscribers:
|
||||
try:
|
||||
msg = sub.recv(zmq.NOBLOCK)
|
||||
msg = msg.decode('utf8')
|
||||
yield msg.split(" ", 1)[1]
|
||||
yield msg.split(b" ", 1)[1]
|
||||
except zmq.error.Again as e:
|
||||
time.sleep(0.2)
|
||||
pass
|
||||
|
@ -131,7 +132,8 @@ class Process(object):
|
|||
self.r_temp = redis.StrictRedis(
|
||||
host=self.config.get('RedisPubSub', 'host'),
|
||||
port=self.config.get('RedisPubSub', 'port'),
|
||||
db=self.config.get('RedisPubSub', 'db'))
|
||||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.moduleNum = os.getpid()
|
||||
|
||||
|
@ -152,11 +154,6 @@ class Process(object):
|
|||
int(self.r_temp.scard(in_set)))
|
||||
message = self.r_temp.spop(in_set)
|
||||
|
||||
try:
|
||||
message = message.decode('utf8')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
|
||||
dir_name = os.environ['AIL_HOME']+self.config.get('Directories', 'pastes')
|
||||
|
||||
|
@ -216,11 +213,6 @@ class Process(object):
|
|||
while True:
|
||||
message = self.r_temp.spop(self.subscriber_name + 'out')
|
||||
|
||||
try:
|
||||
message = message.decode('utf8')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
if message is None:
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -34,7 +34,8 @@ if __name__ == "__main__":
|
|||
r_serv2 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Cache", "host"),
|
||||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
db=p.config.getint("Redis_Cache", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Suscribed to channel mails_categ")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Mixer Module
|
||||
|
@ -65,7 +65,8 @@ if __name__ == '__main__':
|
|||
server = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Mixer_Cache", "host"),
|
||||
port=cfg.getint("Redis_Mixer_Cache", "port"),
|
||||
db=cfg.getint("Redis_Mixer_Cache", "db"))
|
||||
db=cfg.getint("Redis_Mixer_Cache", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# LOGGING #
|
||||
publisher.info("Feed Script started to receive & publish.")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -199,7 +199,8 @@ if __name__ == "__main__":
|
|||
server = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
if args.clear == 1:
|
||||
clearRedisModuleInfo()
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
This module makes statistics for some modules and providers
|
||||
|
@ -52,7 +52,7 @@ def compute_most_posted(server, message):
|
|||
# Member set is a list of (value, score) pairs
|
||||
if int(member_set[0][1]) < keyword_total_sum:
|
||||
#remove min from set and add the new one
|
||||
print(module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0].decode('utf8')+'('+str(member_set[0][1])+')')
|
||||
print(module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
|
||||
server.zrem(redis_progression_name_set, member_set[0][0])
|
||||
server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
|
||||
print(redis_progression_name_set)
|
||||
|
@ -135,12 +135,14 @@ if __name__ == '__main__':
|
|||
r_serv_trend = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_Trending", "host"),
|
||||
port=p.config.get("Redis_Level_DB_Trending", "port"),
|
||||
db=p.config.get("Redis_Level_DB_Trending", "db"))
|
||||
db=p.config.get("Redis_Level_DB_Trending", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_pasteName = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Paste_Name", "host"),
|
||||
port=p.config.get("Redis_Paste_Name", "port"),
|
||||
db=p.config.get("Redis_Paste_Name", "db"))
|
||||
db=p.config.get("Redis_Paste_Name", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
|
||||
|
@ -510,9 +510,8 @@ def clearRedisModuleInfo():
|
|||
|
||||
def cleanRedis():
|
||||
for k in server.keys("MODULE_TYPE_*"):
|
||||
moduleName = (k[12:].decode('utf8')).split('_')[0]
|
||||
moduleName = k[12:].split('_')[0]
|
||||
for pid in server.smembers(k):
|
||||
pid = pid.decode('utf8')
|
||||
flag_pid_valid = False
|
||||
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
||||
try:
|
||||
|
@ -530,7 +529,7 @@ def cleanRedis():
|
|||
#print flag_pid_valid, 'cleaning', pid, 'in', k
|
||||
server.srem(k, pid)
|
||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||
log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k).decode('utf8')], 0))
|
||||
log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k)], 0))
|
||||
|
||||
#Error due to resize, interrupted sys call
|
||||
except IOError as e:
|
||||
|
@ -607,19 +606,16 @@ def fetchQueueData():
|
|||
printarray_idle = []
|
||||
printarray_notrunning = []
|
||||
for queue, card in iter(server.hgetall("queues").items()):
|
||||
queue = queue.decode('utf8')
|
||||
card = card.decode('utf8')
|
||||
all_queue.add(queue)
|
||||
key = "MODULE_" + queue + "_"
|
||||
keySet = "MODULE_TYPE_" + queue
|
||||
array_module_type = []
|
||||
|
||||
for moduleNum in server.smembers(keySet):
|
||||
moduleNum = moduleNum.decode('utf8')
|
||||
value = ( server.get(key + str(moduleNum)) ).decode('utf8')
|
||||
value = server.get(key + str(moduleNum))
|
||||
complete_paste_path = ( server.get(key + str(moduleNum) + "_PATH") )
|
||||
if(complete_paste_path is not None):
|
||||
complete_paste_path = complete_paste_path.decode('utf8')
|
||||
complete_paste_path = complete_paste_path
|
||||
COMPLETE_PASTE_PATH_PER_PID[moduleNum] = complete_paste_path
|
||||
|
||||
if value is not None:
|
||||
|
@ -814,7 +810,8 @@ if __name__ == "__main__":
|
|||
server = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
if args.clear == 1:
|
||||
clearRedisModuleInfo()
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import configparser
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The ZMQ_Sub_Onion Module
|
||||
|
@ -94,7 +94,8 @@ if __name__ == "__main__":
|
|||
r_cache = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Cache", "host"),
|
||||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
db=p.config.getint("Redis_Cache", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script subscribed to channel onion_categ")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import redis
|
||||
|
@ -30,7 +30,8 @@ def main():
|
|||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# LOGGING #
|
||||
publisher.port = 6380
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
This Module is used for term frequency.
|
||||
|
@ -56,7 +56,8 @@ if __name__ == "__main__":
|
|||
server_term = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("RegexForTermsFrequency script started")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
import time
|
||||
from packages import Paste
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python3.5
|
||||
#!/usr/bin/python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import redis
|
||||
|
@ -35,7 +35,8 @@ def main():
|
|||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_Hashs", "host"),
|
||||
port=curYear,
|
||||
db=cfg.getint("Redis_Level_DB_Hashs", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_Hashs", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# LOGGING #
|
||||
publisher.port = 6380
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Sentiment analyser module.
|
||||
|
@ -109,7 +109,7 @@ def Analyse(message, server):
|
|||
provider_timestamp = provider + '_' + str(timestamp)
|
||||
server.incr('UniqID')
|
||||
UniqID = server.get('UniqID')
|
||||
print(provider_timestamp, '->', UniqID.decode('utf8'), 'dropped', num_line_removed, 'lines')
|
||||
print(provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines')
|
||||
server.sadd(provider_timestamp, UniqID)
|
||||
server.set(UniqID, avg_score)
|
||||
else:
|
||||
|
@ -154,7 +154,8 @@ if __name__ == '__main__':
|
|||
server = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_Sentiment", "host"),
|
||||
port=p.config.get("Redis_Level_DB_Sentiment", "port"),
|
||||
db=p.config.get("Redis_Level_DB_Sentiment", "db"))
|
||||
db=p.config.get("Redis_Level_DB_Sentiment", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
while True:
|
||||
message = p.get_from_set()
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
This Module is used for term frequency.
|
||||
|
@ -54,7 +54,8 @@ if __name__ == "__main__":
|
|||
server_term = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
|
||||
db=p.config.get("Redis_Level_DB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("RegexForTermsFrequency script started")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The ZMQ_Feed_Q Module
|
||||
|
@ -37,7 +37,8 @@ def main():
|
|||
# REDIS
|
||||
r_serv = redis.StrictRedis(host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FIXME: automatic based on the queue name.
|
||||
# ### SCRIPTS ####
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
import time
|
||||
from packages import Paste
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Tokenize Module
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import configparser
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -47,7 +47,8 @@ if __name__ == "__main__":
|
|||
r_serv2 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Cache", "host"),
|
||||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
db=p.config.getint("Redis_Cache", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Protocol file path
|
||||
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -116,7 +116,8 @@ if __name__ == '__main__':
|
|||
r_serv_trend = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_Trending", "host"),
|
||||
port=p.config.get("Redis_Level_DB_Trending", "port"),
|
||||
db=p.config.get("Redis_Level_DB_Trending", "db"))
|
||||
db=p.config.get("Redis_Level_DB_Trending", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FILE CURVE SECTION #
|
||||
csv_path_proto = os.path.join(os.environ['AIL_HOME'],
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
|
||||
|
@ -15,7 +15,7 @@ class AilleakObject(AbstractMISPObjectGenerator):
|
|||
self._p_source = p_source.split('/')[-5:]
|
||||
self._p_source = '/'.join(self._p_source)[:-3] # -3 removes .gz
|
||||
self._p_date = p_date
|
||||
self._p_content = p_content.encode('utf8')
|
||||
self._p_content = p_content
|
||||
self._p_duplicate = p_duplicate
|
||||
self._p_duplicate_number = p_duplicate_number
|
||||
self.generate_attributes()
|
||||
|
@ -45,13 +45,10 @@ class ObjectWrapper:
|
|||
self.paste = Paste.Paste(path)
|
||||
self.p_date = self.date_to_str(self.paste.p_date)
|
||||
self.p_source = self.paste.p_path
|
||||
self.p_content = self.paste.get_p_content().decode('utf8')
|
||||
self.p_content = self.paste.get_p_content()
|
||||
|
||||
temp = self.paste._get_p_duplicate()
|
||||
try:
|
||||
temp = temp.decode('utf8')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
#beautifier
|
||||
temp = json.loads(temp)
|
||||
self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
@ -54,7 +54,8 @@ if __name__ == "__main__":
|
|||
server = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"),
|
||||
port=curYear,
|
||||
db=p.config.get("Redis_Level_DB", "db"))
|
||||
db=p.config.get("Redis_Level_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
@ -62,7 +63,6 @@ if __name__ == "__main__":
|
|||
while True:
|
||||
message = p.get_from_set()
|
||||
if message is not None:
|
||||
#decode because of pyhton3
|
||||
module_name, p_path = message.split(';')
|
||||
print("new alert : {}".format(module_name))
|
||||
#PST = Paste.Paste(p_path)
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# This file is part of AIL framework - Analysis Information Leak framework
|
||||
|
@ -50,7 +50,7 @@ socket = context.socket(zmq.PUB)
|
|||
socket.bind(zmq_url)
|
||||
|
||||
# check https://github.com/cvandeplas/pystemon/blob/master/pystemon.yaml#L16
|
||||
r = redis.StrictRedis(host='localhost', db=10)
|
||||
r = redis.StrictRedis(host='localhost', db=10, decode_responses=True)
|
||||
|
||||
# 101 pastes processed feed
|
||||
# 102 raw pastes feed
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# This file is part of AIL framework - Analysis Information Leak framework
|
||||
|
@ -19,14 +19,14 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
|
|||
# 101 Name of the pastes only
|
||||
# 102 Full pastes in raw base64(gz)
|
||||
|
||||
topicfilter = b"102"
|
||||
topicfilter = "102"
|
||||
socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
|
||||
print('b0')
|
||||
|
||||
while True:
|
||||
message = socket.recv()
|
||||
print('b1')
|
||||
print (message)
|
||||
if topicfilter == b"102":
|
||||
if topicfilter == "102":
|
||||
topic, paste, messagedata = message.split()
|
||||
print(paste, messagedata)
|
||||
else:
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import zmq
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# This file is part of AIL framework - Analysis Information Leak framework
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
class Date(object):
|
||||
"""docstring for Date"""
|
||||
def __init__(self, *args):
|
||||
|
@ -38,4 +40,3 @@ class Date(object):
|
|||
comp_month = str(computed_date.month).zfill(2)
|
||||
comp_day = str(computed_date.day).zfill(2)
|
||||
return comp_year + comp_month + comp_day
|
||||
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import hashlib
|
||||
import crcmod
|
||||
import mmh3
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python3.5
|
||||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
The ``Paste Class``
|
||||
|
@ -69,11 +69,13 @@ class Paste(object):
|
|||
self.cache = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
self.store = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Data_Merging", "host"),
|
||||
port=cfg.getint("Redis_Data_Merging", "port"),
|
||||
db=cfg.getint("Redis_Data_Merging", "db"))
|
||||
db=cfg.getint("Redis_Data_Merging", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
self.p_path = p_path
|
||||
self.p_name = os.path.basename(self.p_path)
|
||||
|
@ -112,21 +114,17 @@ class Paste(object):
|
|||
paste = self.cache.get(self.p_path)
|
||||
if paste is None:
|
||||
try:
|
||||
with gzip.open(self.p_path, 'rb') as f:
|
||||
with gzip.open(self.p_path, 'r') as f:
|
||||
paste = f.read()
|
||||
self.cache.set(self.p_path, paste)
|
||||
self.cache.expire(self.p_path, 300)
|
||||
except:
|
||||
paste = b''
|
||||
paste = ''
|
||||
|
||||
return paste.decode('utf8')
|
||||
return paste
|
||||
|
||||
def get_p_content_as_file(self):
|
||||
try:
|
||||
message = StringIO( (self.get_p_content()).decode('utf8') )
|
||||
except AttributeError:
|
||||
message = StringIO( (self.get_p_content()) )
|
||||
|
||||
message = StringIO(self.get_p_content())
|
||||
return message
|
||||
|
||||
def get_p_content_with_removed_lines(self, threshold):
|
||||
|
@ -204,7 +202,7 @@ class Paste(object):
|
|||
|
||||
"""
|
||||
for hash_name, the_hash in self.p_hash_kind.items():
|
||||
self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content().encode('utf8'))
|
||||
self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content().encode())
|
||||
return self.p_hash
|
||||
|
||||
def _get_p_language(self):
|
||||
|
@ -276,7 +274,6 @@ class Paste(object):
|
|||
def _get_p_duplicate(self):
|
||||
self.p_duplicate = self.store.hget(self.p_path, "p_duplicate")
|
||||
if self.p_duplicate is not None:
|
||||
self.p_duplicate = self.p_duplicate.decode('utf8')
|
||||
return self.p_duplicate
|
||||
else:
|
||||
return '[]'
|
||||
|
@ -335,7 +332,7 @@ class Paste(object):
|
|||
json_duplicate = self.store.hget(path, attr_name)
|
||||
#json save on redis
|
||||
if json_duplicate is not None:
|
||||
list_duplicate = json.loads(json_duplicate.decode('utf8'))
|
||||
list_duplicate = json.loads(json_duplicate)
|
||||
# add new duplicate
|
||||
list_duplicate.append([hash_type, self.p_path, percent, date])
|
||||
self.store.hset(path, attr_name, json.dumps(list_duplicate))
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import re
|
||||
import dns.resolver
|
||||
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import os
|
||||
import string
|
||||
|
||||
|
@ -110,7 +112,7 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
|
|||
# if the word have a value for the day
|
||||
# FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold
|
||||
value = r_serv.hget(word, curdate)
|
||||
value = int(value.decode('utf8'))
|
||||
value = int(value)
|
||||
if value >= threshold:
|
||||
row.append(value)
|
||||
writer.writerow(row)
|
||||
|
@ -135,7 +137,7 @@ def create_curve_from_redis_set(server, csvfilename, set_to_plot, year, month):
|
|||
|
||||
redis_set_name = set_to_plot + "_set_" + str(year) + str(month).zfill(2)
|
||||
words = list(server.smembers(redis_set_name))
|
||||
words = [x.decode('utf-8') for x in words]
|
||||
#words = [x.decode('utf-8') for x in words]
|
||||
|
||||
headers = ['Date'] + words
|
||||
with open(csvfilename+'.csv', 'w') as f:
|
||||
|
@ -154,5 +156,5 @@ def create_curve_from_redis_set(server, csvfilename, set_to_plot, year, month):
|
|||
row.append(0)
|
||||
else:
|
||||
# if the word have a value for the day
|
||||
row.append(value.decode('utf8'))
|
||||
row.append(value)
|
||||
writer.writerow(row)
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Template for new modules
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import socks
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import redis
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"Hepler to create a new webpage associated with a module."
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -26,37 +26,44 @@ cfg.read(configfile)
|
|||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"))
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_log = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Log", "host"),
|
||||
port=cfg.getint("Redis_Log", "port"),
|
||||
db=cfg.getint("Redis_Log", "db"))
|
||||
db=cfg.getint("Redis_Log", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_charts = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_Trending", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_Trending", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_Trending", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_Trending", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_sentiment = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_Sentiment", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_Sentiment", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_term = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_TermFreq", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_cred = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_TermCred", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_TermCred", "db"))
|
||||
db=cfg.getint("Redis_Level_DB_TermCred", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_pasteName = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Paste_Name", "host"),
|
||||
port=cfg.getint("Redis_Paste_Name", "port"),
|
||||
db=cfg.getint("Redis_Paste_Name", "db"))
|
||||
db=cfg.getint("Redis_Paste_Name", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# VARIABLES #
|
||||
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -37,7 +37,8 @@ for year in os.listdir(lvdbdir):
|
|||
r_serv_db[intYear] = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB", "host"),
|
||||
port=intYear,
|
||||
db=cfg.getint("Redis_Level_DB", "db"))
|
||||
db=cfg.getint("Redis_Level_DB", "db"),
|
||||
decode_responses=True)
|
||||
yearList.sort(reverse=True)
|
||||
|
||||
browsepastes = Blueprint('browsepastes', __name__, template_folder='templates')
|
||||
|
@ -55,8 +56,8 @@ def getPastebyType(server, module_name):
|
|||
def event_stream_getImportantPasteByModule(module_name, year):
|
||||
index = 0
|
||||
all_pastes_list = getPastebyType(r_serv_db[year], module_name)
|
||||
|
||||
for path in all_pastes_list:
|
||||
path = path.decode('utf8')
|
||||
index += 1
|
||||
paste = Paste.Paste(path)
|
||||
content = paste.get_p_content()
|
||||
|
@ -94,7 +95,6 @@ def importantPasteByModule():
|
|||
allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name)
|
||||
|
||||
for path in allPastes[0:10]:
|
||||
path = path.decode('utf8')
|
||||
all_path.append(path)
|
||||
paste = Paste.Paste(path)
|
||||
content = paste.get_p_content()
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -26,22 +26,10 @@ def event_stream():
|
|||
pubsub = r_serv_log.pubsub()
|
||||
pubsub.psubscribe("Script" + '.*')
|
||||
for msg in pubsub.listen():
|
||||
# bytes conversion
|
||||
try:
|
||||
type = msg['type'].decode('utf8')
|
||||
except:
|
||||
|
||||
type = msg['type']
|
||||
try:
|
||||
pattern = msg['pattern'].decode('utf8')
|
||||
except:
|
||||
pattern = msg['pattern']
|
||||
try:
|
||||
channel = msg['channel'].decode('utf8')
|
||||
except:
|
||||
channel = msg['channel']
|
||||
try:
|
||||
data = msg['data'].decode('utf8')
|
||||
except:
|
||||
data = msg['data']
|
||||
|
||||
msg = {'channel': channel, 'type': type, 'pattern': pattern, 'data': data}
|
||||
|
@ -54,15 +42,13 @@ def get_queues(r):
|
|||
# We may want to put the llen in a pipeline to do only one query.
|
||||
newData = []
|
||||
for queue, card in r.hgetall("queues").items():
|
||||
queue = queue.decode('utf8')
|
||||
card = card.decode('utf8')
|
||||
|
||||
key = "MODULE_" + queue + "_"
|
||||
keySet = "MODULE_TYPE_" + queue
|
||||
|
||||
for moduleNum in r.smembers(keySet):
|
||||
moduleNum = moduleNum.decode('utf8')
|
||||
|
||||
value = ( r.get(key + str(moduleNum)) ).decode('utf8')
|
||||
value = r.get(key + str(moduleNum))
|
||||
|
||||
if value is not None:
|
||||
timestamp, path = value.split(", ")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -58,11 +58,7 @@ def sentiment_analysis_getplotdata():
|
|||
if allProvider == "True":
|
||||
range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
|
||||
range_providers_str = []
|
||||
for domain in range_providers:
|
||||
m = domain.decode('utf8')
|
||||
range_providers_str.append(m)
|
||||
return jsonify(list(range_providers_str))
|
||||
return jsonify(list(range_providers))
|
||||
else:
|
||||
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
|
||||
# if empty, get yesterday top providers
|
||||
|
@ -74,13 +70,7 @@ def sentiment_analysis_getplotdata():
|
|||
print('today provider empty')
|
||||
range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
|
||||
# decode bytes
|
||||
range_providers_str = []
|
||||
for domain in range_providers:
|
||||
m = domain.decode('utf8')
|
||||
range_providers_str.append(m)
|
||||
|
||||
return jsonify(list(range_providers_str))
|
||||
return jsonify(list(range_providers))
|
||||
|
||||
elif provider is not None:
|
||||
to_return = {}
|
||||
|
@ -92,7 +82,7 @@ def sentiment_analysis_getplotdata():
|
|||
|
||||
list_value = []
|
||||
for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
cur_value = (r_serv_sentiment.get(cur_id)).decode('utf8')
|
||||
cur_value = (r_serv_sentiment.get(cur_id))
|
||||
list_value.append(cur_value)
|
||||
list_date[cur_timestamp] = list_value
|
||||
to_return[provider] = list_date
|
||||
|
@ -115,7 +105,7 @@ def sentiment_analysis_plot_tool_getdata():
|
|||
if getProviders == 'True':
|
||||
providers = []
|
||||
for cur_provider in r_serv_charts.smembers('all_provider_set'):
|
||||
providers.append(cur_provider.decode('utf8'))
|
||||
providers.append(cur_provider)
|
||||
return jsonify(providers)
|
||||
|
||||
else:
|
||||
|
@ -144,7 +134,7 @@ def sentiment_analysis_plot_tool_getdata():
|
|||
|
||||
list_value = []
|
||||
for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
cur_value = (r_serv_sentiment.get(cur_id)).decode('utf8')
|
||||
cur_value = (r_serv_sentiment.get(cur_id))
|
||||
list_value.append(cur_value)
|
||||
list_date[cur_timestamp] = list_value
|
||||
to_return[cur_provider] = list_date
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -72,7 +72,7 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
|||
curr_to_return = 0
|
||||
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
|
||||
value = r_serv_term.hget(per_paste+str(timestamp), word)
|
||||
curr_to_return += int(value.decode('utf8')) if value is not None else 0
|
||||
curr_to_return += int(value) if value is not None else 0
|
||||
for i in num_day:
|
||||
if passed_days == i-1:
|
||||
to_return.append(curr_to_return)
|
||||
|
@ -157,10 +157,8 @@ def terms_management():
|
|||
trackReg_list_values = []
|
||||
trackReg_list_num_of_paste = []
|
||||
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
|
||||
tracked_regex = tracked_regex.decode('utf8')
|
||||
print(tracked_regex)
|
||||
|
||||
notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)).decode('utf8') )
|
||||
notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) )
|
||||
|
||||
if tracked_regex not in notificationEnabledDict:
|
||||
notificationEnabledDict[tracked_regex] = False
|
||||
|
@ -176,7 +174,7 @@ def terms_management():
|
|||
value_range.append(term_date)
|
||||
trackReg_list_values.append(value_range)
|
||||
|
||||
if tracked_regex.encode('utf8') in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
notificationEnabledDict[tracked_regex] = True
|
||||
|
||||
#Set
|
||||
|
@ -184,9 +182,9 @@ def terms_management():
|
|||
trackSet_list_values = []
|
||||
trackSet_list_num_of_paste = []
|
||||
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
|
||||
tracked_set = tracked_set.decode('utf8')
|
||||
tracked_set = tracked_set
|
||||
|
||||
notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)).decode('utf8') )
|
||||
notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) )
|
||||
|
||||
|
||||
if tracked_set not in notificationEnabledDict:
|
||||
|
@ -203,7 +201,7 @@ def terms_management():
|
|||
value_range.append(term_date)
|
||||
trackSet_list_values.append(value_range)
|
||||
|
||||
if tracked_set.encode('utf8') in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
notificationEnabledDict[tracked_set] = True
|
||||
|
||||
#Tracked terms
|
||||
|
@ -211,13 +209,7 @@ def terms_management():
|
|||
track_list_values = []
|
||||
track_list_num_of_paste = []
|
||||
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
|
||||
tracked_term = tracked_term.decode('utf8')
|
||||
print('tracked_term : .')
|
||||
print(tracked_term)
|
||||
|
||||
#print(TrackedTermsNotificationEmailsPrefix_Name)
|
||||
print(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
|
||||
#print(tracked_term)
|
||||
notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
|
||||
|
||||
if tracked_term not in notificationEnabledDict:
|
||||
|
@ -229,16 +221,14 @@ def terms_management():
|
|||
term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
|
||||
|
||||
set_paste_name = "tracked_" + tracked_term
|
||||
print('set_paste_name : .')
|
||||
print(set_paste_name)
|
||||
|
||||
track_list_num_of_paste.append( r_serv_term.scard(set_paste_name) )
|
||||
print('track_list_num_of_paste : .')
|
||||
print(track_list_num_of_paste)
|
||||
|
||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
value_range.append(term_date)
|
||||
track_list_values.append(value_range)
|
||||
|
||||
if tracked_term.encode('utf8') in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
||||
notificationEnabledDict[tracked_term] = True
|
||||
|
||||
#blacklist terms
|
||||
|
@ -246,7 +236,7 @@ def terms_management():
|
|||
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
|
||||
term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
|
||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
black_list.append([blacked_term.decode('utf8'), term_date])
|
||||
black_list.append([blacked_term, term_date])
|
||||
|
||||
return render_template("terms_management.html",
|
||||
black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
|
||||
|
@ -259,8 +249,6 @@ def terms_management():
|
|||
@terms.route("/terms_management_query_paste/")
|
||||
def terms_management_query_paste():
|
||||
term = request.args.get('term')
|
||||
print('term :')
|
||||
print(term)
|
||||
paste_info = []
|
||||
|
||||
# check if regex or not
|
||||
|
@ -275,7 +263,6 @@ def terms_management_query_paste():
|
|||
track_list_path = r_serv_term.smembers(set_paste_name)
|
||||
|
||||
for path in track_list_path:
|
||||
path = path.decode('utf8')
|
||||
paste = Paste.Paste(path)
|
||||
p_date = str(paste._get_p_date())
|
||||
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
|
||||
|
@ -402,7 +389,6 @@ def terms_management_action():
|
|||
r_serv_term.hdel(TrackedRegexDate_Name, term)
|
||||
elif term.startswith('\\') and term.endswith('\\'):
|
||||
r_serv_term.srem(TrackedSetSet_Name, term)
|
||||
#print(term)
|
||||
r_serv_term.hdel(TrackedSetDate_Name, term)
|
||||
else:
|
||||
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
|
||||
|
@ -515,7 +501,7 @@ def terms_plot_top_data():
|
|||
curr_value_range = int(value) if value is not None else 0
|
||||
value_range.append([timestamp, curr_value_range])
|
||||
|
||||
to_return.append([term.decode('utf8'), value_range, tot_value, position])
|
||||
to_return.append([term, value_range, tot_value, position])
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
|
@ -532,7 +518,6 @@ def credentials_management_query_paste():
|
|||
paste_info = []
|
||||
for pathNum in allPath:
|
||||
path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
|
||||
path = path.decode('utf8')
|
||||
paste = Paste.Paste(path)
|
||||
p_date = str(paste._get_p_date())
|
||||
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
|
||||
|
@ -574,7 +559,6 @@ def cred_management_action():
|
|||
iter_num = 0
|
||||
tot_iter = len(AllUsernameInRedis)*len(possibilities)
|
||||
for tempUsername in AllUsernameInRedis:
|
||||
tempUsername = tempUsername.decode('utf8')
|
||||
for poss in possibilities:
|
||||
#FIXME print progress
|
||||
if(iter_num % int(tot_iter/20) == 0):
|
||||
|
@ -583,7 +567,7 @@ def cred_management_action():
|
|||
iter_num += 1
|
||||
|
||||
if poss in tempUsername:
|
||||
num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername)).decode('utf8')
|
||||
num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername))
|
||||
if num is not None:
|
||||
uniq_num_set.add(num)
|
||||
for num in r_serv_cred.smembers(tempUsername):
|
||||
|
@ -592,7 +576,7 @@ def cred_management_action():
|
|||
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
|
||||
for Unum in uniq_num_set:
|
||||
levenRatio = 2.0
|
||||
username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)).decode('utf8')
|
||||
username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
|
||||
|
||||
# Calculate Levenshtein distance, ignore negative ratio
|
||||
supp_splitted = supplied.split()
|
||||
|
@ -604,20 +588,11 @@ def cred_management_action():
|
|||
|
||||
data['usr'].append(username)
|
||||
|
||||
try:
|
||||
Unum = Unum.decode('utf8')
|
||||
except:
|
||||
pass
|
||||
|
||||
allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
|
||||
|
||||
# decode bytes
|
||||
allPathNum_str = []
|
||||
for p in allPathNum:
|
||||
allPathNum_str.append(p.decode('utf8'))
|
||||
|
||||
data['path'].append(allPathNum_str)
|
||||
data['numPaste'].append(len(allPathNum_str))
|
||||
data['path'].append(allPathNum)
|
||||
data['numPaste'].append(len(allPathNum))
|
||||
data['simil'].append(levenRatioStr)
|
||||
|
||||
to_return = {}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -47,9 +47,8 @@ def progressionCharts():
|
|||
date_range = get_date_range(num_day)
|
||||
# Retreive all data from the last num_day
|
||||
for date in date_range:
|
||||
|
||||
curr_value = r_serv_charts.hget(attribute_name, date)
|
||||
if curr_value is not None:
|
||||
curr_value = curr_value.decode('utf8')
|
||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||
bar_values.insert(0, attribute_name)
|
||||
return jsonify(bar_values)
|
||||
|
@ -58,13 +57,7 @@ def progressionCharts():
|
|||
redis_progression_name = "z_top_progression_" + trending_name
|
||||
keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
|
||||
|
||||
# decode bytes
|
||||
keyw_value_str = []
|
||||
for domain, value in keyw_value:
|
||||
m = domain.decode('utf8'), value
|
||||
keyw_value_str.append(m)
|
||||
|
||||
return jsonify(keyw_value_str)
|
||||
return jsonify(keyw_value)
|
||||
|
||||
@trendings.route("/wordstrending/")
|
||||
def wordstrending():
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3.5
|
||||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
|
@ -28,18 +28,12 @@ def get_top_relevant_data(server, module_name):
|
|||
for date in get_date_range(15):
|
||||
redis_progression_name_set = 'top_'+ module_name +'_set_' + date
|
||||
member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
|
||||
member_set_str = []
|
||||
|
||||
# decode bytes
|
||||
for domain, value in member_set:
|
||||
m = domain.decode('utf8'), value
|
||||
member_set_str.append(m)
|
||||
|
||||
if len(member_set_str) == 0: #No data for this date
|
||||
if len(member_set) == 0: #No data for this date
|
||||
days += 1
|
||||
else:
|
||||
member_set_str.insert(0, ("passed_days", days))
|
||||
return member_set_str
|
||||
member_set.insert(0, ("passed_days", days))
|
||||
return member_set
|
||||
|
||||
|
||||
def get_date_range(num_day):
|
||||
|
@ -94,13 +88,13 @@ def providersChart():
|
|||
for date in date_range:
|
||||
curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
|
||||
if curr_value_size is not None:
|
||||
curr_value_size = curr_value_size.decode('utf8')
|
||||
curr_value_size = curr_value_size
|
||||
|
||||
curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
|
||||
|
||||
curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
|
||||
if curr_value_size_avg is not None:
|
||||
curr_value_size_avg = curr_value_size_avg.decode('utf8')
|
||||
curr_value_size_avg = curr_value_size_avg
|
||||
|
||||
|
||||
if module_name == "size":
|
||||
|
@ -119,16 +113,10 @@ def providersChart():
|
|||
|
||||
member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
|
||||
|
||||
# decode bytes
|
||||
member_set_str = []
|
||||
for domain, value in member_set:
|
||||
m = domain.decode('utf8'), value
|
||||
member_set_str.append(m)
|
||||
|
||||
# Member set is a list of (value, score) pairs
|
||||
if len(member_set_str) == 0:
|
||||
if len(member_set) == 0:
|
||||
member_set_str.append(("No relevant data", float(100)))
|
||||
return jsonify(member_set_str)
|
||||
return jsonify(member_set)
|
||||
|
||||
|
||||
@trendingmodules.route("/moduletrending/")
|
||||
|
|
Loading…
Reference in a new issue