ail-framework/bin/Web.py

185 lines
6.4 KiB
Python
Raw Normal View History

2018-05-04 11:53:29 +00:00
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Web Module
============================
This module tries to parse URLs and warns if some defined contry code are present.
"""
2021-04-02 07:52:05 +00:00
##################################
# Import External packages
##################################
2014-08-14 15:55:18 +00:00
import redis
import pprint
import time
import os
2014-08-11 07:27:50 +00:00
import dns.exception
from pyfaup.faup import Faup
import re
# Country and ASN lookup
from cymru.ip2asn.dns import DNSClient as ip2asn
import socket
import pycountry
import ipaddress
2021-04-02 07:52:05 +00:00
##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from packages import Paste
from packages import lib_refine
from Helper import Process
2018-04-16 15:00:44 +00:00
2021-04-02 07:52:05 +00:00
class Web(AbstractModule):
"""
Web module for AIL framework
"""
# Used to prevent concat with empty fields due to url parsing
def avoidNone(self, a_string):
if a_string is None:
return ""
else:
return a_string
def __init__(self):
"""
Init Web
"""
2021-04-28 13:24:33 +00:00
super(Web, self).__init__(logger_channel='script:web')
2021-04-02 07:52:05 +00:00
# REDIS Cache
self.r_serv2 = redis.StrictRedis(
host=self.process.config.get("Redis_Cache", "host"),
port=self.process.config.getint("Redis_Cache", "port"),
db=self.process.config.getint("Redis_Cache", "db"),
decode_responses=True)
# Country to log as critical
self.cc_critical = self.process.config.get("Url", "cc_critical")
# FUNCTIONS #
self.faup = Faup()
# Protocol file path
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
self.process.config.get("Directories", "protocolsfile"))
# Get all uri from protocolsfile (Used for Curve)
uri_scheme = ""
with open(protocolsfile_path, 'r') as scheme_file:
for scheme in scheme_file:
uri_scheme += scheme[:-1]+"|"
uri_scheme = uri_scheme[:-1]
self.url_regex = "((?i:"+uri_scheme + \
")\://(?:[a-zA-Z0-9\.\-]+(?:\:[a-zA-Z0-9\.&%\$\-]+)*@)*(?:(?:25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(?:25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|(?:[a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(?:com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)"
self.prec_filename = None
# Send module state to logs
2021-04-28 13:24:33 +00:00
self.redis_logger.info(f"Module {self.module_name} initialized")
2021-04-02 07:52:05 +00:00
def compute(self, message):
"""
Search for Web links from given message
"""
# Extract item
filename, score = message.split()
2021-04-28 13:24:33 +00:00
domains_list = set()
hosts_list = set()
2021-04-02 07:52:05 +00:00
if self.prec_filename is None or filename != self.prec_filename:
2021-04-28 13:24:33 +00:00
domains_list.clear()
hosts_list.clear()
2021-04-02 07:52:05 +00:00
PST = Paste.Paste(filename)
client = ip2asn()
detected_urls = PST.get_regex(self.url_regex)
if len(detected_urls) > 0:
2021-04-28 13:24:33 +00:00
to_print = f'Web;{PST.p_source};{PST.p_date};{PST.p_name};'
self.redis_logger.info(f'{to_print}Detected {len(detected_urls)} URL;{PST.p_rel_path}')
2021-04-02 07:52:05 +00:00
for url in detected_urls:
2021-04-28 13:24:33 +00:00
if url.endswith(".on"):
# URL is an onion link skip
# TODO send to TOR crawler ?
# self.redis_logger.debug("Skip onion link")
continue
2021-04-02 07:52:05 +00:00
2021-04-28 13:24:33 +00:00
self.redis_logger.debug(f"match regex: {url}")
2021-04-02 07:52:05 +00:00
2021-04-28 13:24:33 +00:00
to_send = f"{url} {PST._get_p_date()} {filename}"
2021-04-02 07:52:05 +00:00
self.process.populate_set_out(to_send, 'Url')
2021-04-28 13:24:33 +00:00
self.redis_logger.debug(f"url_parsed: {to_send}")
2021-04-02 07:52:05 +00:00
self.faup.decode(url)
domain = self.faup.get_domain()
subdomain = self.faup.get_subdomain()
2021-04-28 13:24:33 +00:00
self.redis_logger.debug(f'{url} Published')
2021-04-02 07:52:05 +00:00
2021-04-28 13:24:33 +00:00
domains_list.add(domain)
hostl = f'{subdomain}.{domain}' if subdomain else domain
if hostl not in hosts_list:
# test host only once a host in a paste
hosts_list.add(hostl)
2014-09-04 09:46:07 +00:00
try:
2021-04-28 13:24:33 +00:00
socket.setdefaulttimeout(1)
ip = socket.gethostbyname(hostl)
# If the resolver is not giving any IPv4 address,
# ASN/CC lookup is skip.
l = client.lookup(ip, qType='IP')
except ipaddress.AddressValueError:
self.redis_logger.debug(
f'ASN/CC lookup failed for IP {ip}')
continue
2021-04-02 07:52:05 +00:00
except:
2021-04-28 13:24:33 +00:00
self.redis_logger.debug(
f'Resolver IPv4 address failed for host {hostl}')
continue
cc = getattr(l, 'cc')
asn = ''
if getattr(l, 'asn') is not None:
asn = getattr(l, 'asn')[2:] # remobe b'
# EU is not an official ISO 3166 code (but used by RIPE
# IP allocation)
if cc is not None and cc != "EU":
countryname = pycountry.countries.get(alpha_2=cc).name
self.redis_logger.debug(f'{hostl};{asn};{cc};{countryname}')
if cc == self.cc_critical:
to_print = f'Url;{PST.p_source};{PST.p_date};{PST.p_name};Detected {hostl} {cc}'
self.redis_logger.info(to_print)
else:
self.redis_logger.debug(f'{hostl};{asn};{cc}')
2021-04-02 07:52:05 +00:00
A_values = lib_refine.checking_A_record(self.r_serv2,
domains_list)
if A_values[0] >= 1:
pprint.pprint(A_values)
# self.redis_logger.info('Url;{};{};{};Checked {} URL;{}'.format(
# PST.p_source, PST.p_date, PST.p_name, A_values[0], PST.p_rel_path))
self.prec_filename = filename
if __name__ == '__main__':
module = Web()
module.run()