mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
Fix the exceptions
This commit is contained in:
Raphaël Vinot
parent
9e8611a42d
commit
b7c9e489c9
3 changed files with 100 additions and 98 deletions
55
bin/Mail.py
55
bin/Mail.py
|
|
@ -38,40 +38,35 @@ if __name__ == "__main__":
|
||||||
is_critical = 10
|
is_critical = 10
|
||||||
|
|
||||||
email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}"
|
email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}"
|
||||||
|
MX_values = None
|
||||||
while True:
|
while True:
|
||||||
try:
|
if message is not None:
|
||||||
if message is not None:
|
filename, word, score = message.split()
|
||||||
print message
|
|
||||||
filename, word, score = message.split()
|
|
||||||
|
|
||||||
if prec_filename is None or filename != prec_filename:
|
if prec_filename is None or filename != prec_filename:
|
||||||
PST = Paste.Paste(filename)
|
PST = Paste.Paste(filename)
|
||||||
MX_values = lib_refine.checking_MX_record(
|
MX_values = lib_refine.checking_MX_record(
|
||||||
r_serv2, PST.get_regex(email_regex))
|
r_serv2, PST.get_regex(email_regex))
|
||||||
|
|
||||||
if MX_values[0] >= 1:
|
if MX_values[0] >= 1:
|
||||||
|
|
||||||
PST.__setattr__(channel, MX_values)
|
PST.__setattr__(channel, MX_values)
|
||||||
PST.save_attribute_redis(channel, (MX_values[0],
|
PST.save_attribute_redis(channel, (MX_values[0],
|
||||||
list(MX_values[1])))
|
list(MX_values[1])))
|
||||||
|
|
||||||
pprint.pprint(MX_values)
|
pprint.pprint(MX_values)
|
||||||
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
|
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
|
||||||
format(PST.p_source, PST.p_date, PST.p_name,
|
format(PST.p_source, PST.p_date, PST.p_name,
|
||||||
MX_values[0])
|
MX_values[0])
|
||||||
if MX_values[0] > is_critical:
|
if MX_values[0] > is_critical:
|
||||||
publisher.warning(to_print)
|
publisher.warning(to_print)
|
||||||
else:
|
else:
|
||||||
publisher.info(to_print)
|
publisher.info(to_print)
|
||||||
prec_filename = filename
|
prec_filename = filename
|
||||||
|
|
||||||
else:
|
else:
|
||||||
publisher.debug("Script Mails is Idling 10s")
|
publisher.debug("Script Mails is Idling 10s")
|
||||||
print 'Sleeping'
|
print 'Sleeping'
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
message = p.get_from_set()
|
message = p.get_from_set()
|
||||||
except dns.exception.Timeout:
|
|
||||||
# FIXME retry!
|
|
||||||
print "dns.exception.Timeout"
|
|
||||||
|
|
|
||||||
115
bin/Url.py
115
bin/Url.py
|
|
@ -45,71 +45,68 @@ if __name__ == "__main__":
|
||||||
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
if message is not None:
|
||||||
if message is not None:
|
filename, word, score = message.split()
|
||||||
filename, word, score = message.split()
|
|
||||||
|
|
||||||
if prec_filename is None or filename != prec_filename:
|
if prec_filename is None or filename != prec_filename:
|
||||||
domains_list = []
|
domains_list = []
|
||||||
PST = Paste.Paste(filename)
|
PST = Paste.Paste(filename)
|
||||||
client = ip2asn()
|
client = ip2asn()
|
||||||
for x in PST.get_regex(url_regex):
|
for x in PST.get_regex(url_regex):
|
||||||
scheme, credential, subdomain, domain, host, tld, \
|
scheme, credential, subdomain, domain, host, tld, \
|
||||||
port, resource_path, query_string, f1, f2, f3, \
|
port, resource_path, query_string, f1, f2, f3, \
|
||||||
f4 = x
|
f4 = x
|
||||||
domains_list.append(domain)
|
domains_list.append(domain)
|
||||||
p.populate_set_out(x, 'Url')
|
p.populate_set_out(x, 'Url')
|
||||||
publisher.debug('{} Published'.format(x))
|
publisher.debug('{} Published'.format(x))
|
||||||
|
|
||||||
if f1 == "onion":
|
if f1 == "onion":
|
||||||
print domain
|
print domain
|
||||||
|
|
||||||
hostl = unicode(subdomain+domain)
|
hostl = unicode(subdomain+domain)
|
||||||
try:
|
try:
|
||||||
socket.setdefaulttimeout(2)
|
socket.setdefaulttimeout(2)
|
||||||
ip = socket.gethostbyname(unicode(hostl))
|
ip = socket.gethostbyname(unicode(hostl))
|
||||||
except:
|
except:
|
||||||
# If the resolver is not giving any IPv4 address,
|
# If the resolver is not giving any IPv4 address,
|
||||||
# ASN/CC lookup is skip.
|
# ASN/CC lookup is skip.
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
l = client.lookup(ip, qType='IP')
|
l = client.lookup(ip, qType='IP')
|
||||||
except ipaddress.AddressValueError:
|
except ipaddress.AddressValueError:
|
||||||
continue
|
continue
|
||||||
cc = getattr(l, 'cc')
|
cc = getattr(l, 'cc')
|
||||||
asn = getattr(l, 'asn')
|
asn = getattr(l, 'asn')
|
||||||
|
|
||||||
# EU is not an official ISO 3166 code (but used by RIPE
|
# EU is not an official ISO 3166 code (but used by RIPE
|
||||||
# IP allocation)
|
# IP allocation)
|
||||||
if cc is not None and cc != "EU":
|
if cc is not None and cc != "EU":
|
||||||
print hostl, asn, cc, \
|
print hostl, asn, cc, \
|
||||||
pycountry.countries.get(alpha2=cc).name
|
pycountry.countries.get(alpha2=cc).name
|
||||||
if cc == cc_critical:
|
if cc == cc_critical:
|
||||||
publisher.warning(
|
publisher.warning(
|
||||||
'Url;{};{};{};Detected {} {}'.format(
|
'Url;{};{};{};Detected {} {}'.format(
|
||||||
PST.p_source, PST.p_date, PST.p_name,
|
PST.p_source, PST.p_date, PST.p_name,
|
||||||
hostl, cc))
|
hostl, cc))
|
||||||
else:
|
else:
|
||||||
print hostl, asn, cc
|
print hostl, asn, cc
|
||||||
|
|
||||||
A_values = lib_refine.checking_A_record(r_serv2,
|
A_values = lib_refine.checking_A_record(r_serv2,
|
||||||
domains_list)
|
domains_list)
|
||||||
if A_values[0] >= 1:
|
if A_values[0] >= 1:
|
||||||
PST.__setattr__(channel, A_values)
|
PST.__setattr__(channel, A_values)
|
||||||
PST.save_attribute_redis(channel, (A_values[0],
|
PST.save_attribute_redis(channel, (A_values[0],
|
||||||
list(A_values[1])))
|
list(A_values[1])))
|
||||||
|
|
||||||
pprint.pprint(A_values)
|
pprint.pprint(A_values)
|
||||||
publisher.info('Url;{};{};{};Checked {} URL'.format(
|
publisher.info('Url;{};{};{};Checked {} URL'.format(
|
||||||
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
|
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
|
||||||
prec_filename = filename
|
prec_filename = filename
|
||||||
|
|
||||||
else:
|
else:
|
||||||
publisher.debug("Script url is Idling 10s")
|
publisher.debug("Script url is Idling 10s")
|
||||||
print 'Sleeping'
|
print 'Sleeping'
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
message = p.get_from_set()
|
message = p.get_from_set()
|
||||||
except dns.exception.Timeout:
|
|
||||||
print "dns.exception.Timeout", A_values
|
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,9 @@ def checking_MX_record(r_serv, adress_set):
|
||||||
WalidMX = set([])
|
WalidMX = set([])
|
||||||
# Transforming the set into a string
|
# Transforming the set into a string
|
||||||
MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower())
|
MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower())
|
||||||
|
resolver = dns.resolver.Resolver()
|
||||||
|
resolver.timeout = 5
|
||||||
|
resolver.lifetime = 2
|
||||||
if MXdomains != []:
|
if MXdomains != []:
|
||||||
|
|
||||||
for MXdomain in set(MXdomains):
|
for MXdomain in set(MXdomains):
|
||||||
|
|
@ -45,9 +47,9 @@ def checking_MX_record(r_serv, adress_set):
|
||||||
# Not already in Redis
|
# Not already in Redis
|
||||||
else:
|
else:
|
||||||
# If I'm Walid MX domain
|
# If I'm Walid MX domain
|
||||||
if dns.resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
|
if resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
|
||||||
# Gonna be added in redis.
|
# Gonna be added in redis.
|
||||||
r_serv.setex(MXdomain[1:], timedelta(days=1), 1)
|
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||||
score += 1
|
score += 1
|
||||||
WalidMX.add(MXdomain[1:])
|
WalidMX.add(MXdomain[1:])
|
||||||
else:
|
else:
|
||||||
|
|
@ -63,13 +65,17 @@ def checking_MX_record(r_serv, adress_set):
|
||||||
publisher.debug('SyntaxError: EmptyLabel')
|
publisher.debug('SyntaxError: EmptyLabel')
|
||||||
|
|
||||||
except dns.resolver.NXDOMAIN:
|
except dns.resolver.NXDOMAIN:
|
||||||
|
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||||
publisher.debug('The query name does not exist.')
|
publisher.debug('The query name does not exist.')
|
||||||
|
|
||||||
except dns.name.LabelTooLong:
|
except dns.name.LabelTooLong:
|
||||||
publisher.debug('The Label is too long')
|
publisher.debug('The Label is too long')
|
||||||
|
|
||||||
finally:
|
except dns.resolver.Timeout:
|
||||||
pass
|
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print e
|
||||||
|
|
||||||
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
|
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
|
||||||
return (num, WalidMX)
|
return (num, WalidMX)
|
||||||
|
|
@ -79,6 +85,9 @@ def checking_A_record(r_serv, domains_set):
|
||||||
score = 0
|
score = 0
|
||||||
num = len(domains_set)
|
num = len(domains_set)
|
||||||
WalidA = set([])
|
WalidA = set([])
|
||||||
|
resolver = dns.resolver.Resolver()
|
||||||
|
resolver.timeout = 5
|
||||||
|
resolver.lifetime = 2
|
||||||
|
|
||||||
for Adomain in domains_set:
|
for Adomain in domains_set:
|
||||||
try:
|
try:
|
||||||
|
|
@ -89,9 +98,9 @@ def checking_A_record(r_serv, domains_set):
|
||||||
# Not already in Redis
|
# Not already in Redis
|
||||||
else:
|
else:
|
||||||
# If I'm Walid domain
|
# If I'm Walid domain
|
||||||
if dns.resolver.query(Adomain, rdtype=dns.rdatatype.A):
|
if resolver.query(Adomain, rdtype=dns.rdatatype.A):
|
||||||
# Gonna be added in redis.
|
# Gonna be added in redis.
|
||||||
r_serv.setex(Adomain, timedelta(days=1), 1)
|
r_serv.setex(Adomain, 1, timedelta(days=1))
|
||||||
score += 1
|
score += 1
|
||||||
WalidA.add(Adomain)
|
WalidA.add(Adomain)
|
||||||
else:
|
else:
|
||||||
|
|
@ -107,13 +116,14 @@ def checking_A_record(r_serv, domains_set):
|
||||||
publisher.debug('SyntaxError: EmptyLabel')
|
publisher.debug('SyntaxError: EmptyLabel')
|
||||||
|
|
||||||
except dns.resolver.NXDOMAIN:
|
except dns.resolver.NXDOMAIN:
|
||||||
|
r_serv.setex(Adomain[1:], 1, timedelta(days=1))
|
||||||
publisher.debug('The query name does not exist.')
|
publisher.debug('The query name does not exist.')
|
||||||
|
|
||||||
except dns.name.LabelTooLong:
|
except dns.name.LabelTooLong:
|
||||||
publisher.debug('The Label is too long')
|
publisher.debug('The Label is too long')
|
||||||
|
|
||||||
finally:
|
except Exception as e:
|
||||||
pass
|
print e
|
||||||
|
|
||||||
publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
|
publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
|
||||||
return (num, WalidA)
|
return (num, WalidA)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue