Fix the exceptions

This commit is contained in:
Raphaël Vinot 2014-09-04 11:46:07 +02:00
parent 9e8611a42d
commit b7c9e489c9
3 changed files with 100 additions and 98 deletions

View file

@ -38,40 +38,35 @@ if __name__ == "__main__":
is_critical = 10
email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}"
MX_values = None
while True:
try:
if message is not None:
print message
filename, word, score = message.split()
if message is not None:
filename, word, score = message.split()
if prec_filename is None or filename != prec_filename:
PST = Paste.Paste(filename)
MX_values = lib_refine.checking_MX_record(
r_serv2, PST.get_regex(email_regex))
if prec_filename is None or filename != prec_filename:
PST = Paste.Paste(filename)
MX_values = lib_refine.checking_MX_record(
r_serv2, PST.get_regex(email_regex))
if MX_values[0] >= 1:
if MX_values[0] >= 1:
PST.__setattr__(channel, MX_values)
PST.save_attribute_redis(channel, (MX_values[0],
list(MX_values[1])))
PST.__setattr__(channel, MX_values)
PST.save_attribute_redis(channel, (MX_values[0],
list(MX_values[1])))
pprint.pprint(MX_values)
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
format(PST.p_source, PST.p_date, PST.p_name,
MX_values[0])
if MX_values[0] > is_critical:
publisher.warning(to_print)
else:
publisher.info(to_print)
prec_filename = filename
pprint.pprint(MX_values)
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
format(PST.p_source, PST.p_date, PST.p_name,
MX_values[0])
if MX_values[0] > is_critical:
publisher.warning(to_print)
else:
publisher.info(to_print)
prec_filename = filename
else:
publisher.debug("Script Mails is Idling 10s")
print 'Sleeping'
time.sleep(10)
else:
publisher.debug("Script Mails is Idling 10s")
print 'Sleeping'
time.sleep(10)
message = p.get_from_set()
except dns.exception.Timeout:
# FIXME retry!
print "dns.exception.Timeout"
message = p.get_from_set()

View file

@ -45,71 +45,68 @@ if __name__ == "__main__":
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
while True:
try:
if message is not None:
filename, word, score = message.split()
if message is not None:
filename, word, score = message.split()
if prec_filename is None or filename != prec_filename:
domains_list = []
PST = Paste.Paste(filename)
client = ip2asn()
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 = x
domains_list.append(domain)
p.populate_set_out(x, 'Url')
publisher.debug('{} Published'.format(x))
if prec_filename is None or filename != prec_filename:
domains_list = []
PST = Paste.Paste(filename)
client = ip2asn()
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 = x
domains_list.append(domain)
p.populate_set_out(x, 'Url')
publisher.debug('{} Published'.format(x))
if f1 == "onion":
print domain
if f1 == "onion":
print domain
hostl = unicode(subdomain+domain)
try:
socket.setdefaulttimeout(2)
ip = socket.gethostbyname(unicode(hostl))
except:
# If the resolver is not giving any IPv4 address,
# ASN/CC lookup is skip.
continue
hostl = unicode(subdomain+domain)
try:
socket.setdefaulttimeout(2)
ip = socket.gethostbyname(unicode(hostl))
except:
# If the resolver is not giving any IPv4 address,
# ASN/CC lookup is skip.
continue
try:
l = client.lookup(ip, qType='IP')
except ipaddress.AddressValueError:
continue
cc = getattr(l, 'cc')
asn = getattr(l, 'asn')
try:
l = client.lookup(ip, qType='IP')
except ipaddress.AddressValueError:
continue
cc = getattr(l, 'cc')
asn = getattr(l, 'asn')
# EU is not an official ISO 3166 code (but used by RIPE
# IP allocation)
if cc is not None and cc != "EU":
print hostl, asn, cc, \
pycountry.countries.get(alpha2=cc).name
if cc == cc_critical:
publisher.warning(
'Url;{};{};{};Detected {} {}'.format(
PST.p_source, PST.p_date, PST.p_name,
hostl, cc))
else:
print hostl, asn, cc
# EU is not an official ISO 3166 code (but used by RIPE
# IP allocation)
if cc is not None and cc != "EU":
print hostl, asn, cc, \
pycountry.countries.get(alpha2=cc).name
if cc == cc_critical:
publisher.warning(
'Url;{};{};{};Detected {} {}'.format(
PST.p_source, PST.p_date, PST.p_name,
hostl, cc))
else:
print hostl, asn, cc
A_values = lib_refine.checking_A_record(r_serv2,
domains_list)
if A_values[0] >= 1:
PST.__setattr__(channel, A_values)
PST.save_attribute_redis(channel, (A_values[0],
list(A_values[1])))
A_values = lib_refine.checking_A_record(r_serv2,
domains_list)
if A_values[0] >= 1:
PST.__setattr__(channel, A_values)
PST.save_attribute_redis(channel, (A_values[0],
list(A_values[1])))
pprint.pprint(A_values)
publisher.info('Url;{};{};{};Checked {} URL'.format(
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
prec_filename = filename
pprint.pprint(A_values)
publisher.info('Url;{};{};{};Checked {} URL'.format(
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
prec_filename = filename
else:
publisher.debug("Script url is Idling 10s")
print 'Sleeping'
time.sleep(10)
else:
publisher.debug("Script url is Idling 10s")
print 'Sleeping'
time.sleep(10)
message = p.get_from_set()
except dns.exception.Timeout:
print "dns.exception.Timeout", A_values
message = p.get_from_set()

View file

@ -33,7 +33,9 @@ def checking_MX_record(r_serv, adress_set):
WalidMX = set([])
# Transforming the set into a string
MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower())
resolver = dns.resolver.Resolver()
resolver.timeout = 5
resolver.lifetime = 2
if MXdomains != []:
for MXdomain in set(MXdomains):
@ -45,9 +47,9 @@ def checking_MX_record(r_serv, adress_set):
# Not already in Redis
else:
# If I'm Walid MX domain
if dns.resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
if resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
# Gonna be added in redis.
r_serv.setex(MXdomain[1:], timedelta(days=1), 1)
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
score += 1
WalidMX.add(MXdomain[1:])
else:
@ -63,13 +65,17 @@ def checking_MX_record(r_serv, adress_set):
publisher.debug('SyntaxError: EmptyLabel')
except dns.resolver.NXDOMAIN:
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
publisher.debug('The query name does not exist.')
except dns.name.LabelTooLong:
publisher.debug('The Label is too long')
finally:
pass
except dns.resolver.Timeout:
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
except Exception as e:
print e
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
return (num, WalidMX)
@ -79,6 +85,9 @@ def checking_A_record(r_serv, domains_set):
score = 0
num = len(domains_set)
WalidA = set([])
resolver = dns.resolver.Resolver()
resolver.timeout = 5
resolver.lifetime = 2
for Adomain in domains_set:
try:
@ -89,9 +98,9 @@ def checking_A_record(r_serv, domains_set):
# Not already in Redis
else:
# If I'm Walid domain
if dns.resolver.query(Adomain, rdtype=dns.rdatatype.A):
if resolver.query(Adomain, rdtype=dns.rdatatype.A):
# Gonna be added in redis.
r_serv.setex(Adomain, timedelta(days=1), 1)
r_serv.setex(Adomain, 1, timedelta(days=1))
score += 1
WalidA.add(Adomain)
else:
@ -107,13 +116,14 @@ def checking_A_record(r_serv, domains_set):
publisher.debug('SyntaxError: EmptyLabel')
except dns.resolver.NXDOMAIN:
r_serv.setex(Adomain[1:], 1, timedelta(days=1))
publisher.debug('The query name does not exist.')
except dns.name.LabelTooLong:
publisher.debug('The Label is too long')
finally:
pass
except Exception as e:
print e
publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
return (num, WalidA)