AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-09-19 23:48:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chg: [crawler] auto tag crawled domains

Browse source
This commit is contained in:
Terrtia 2023-06-20 08:11:44 +02:00
parent c02b9f2a81
commit 501d10bbbd
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
6 changed files with 121 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
bin/crawlers/Crawler.py
View file

@ -200,10 +200,13 @@ class Crawler(AbstractModule):
self.save_capture_response(parent_id, entries)
self.domain.update_daterange(self.date.replace('/', ''))
# Origin + History
# Origin + History + tags
if self.root_item:
self.domain.set_last_origin(parent_id)
self.domain.add_history(epoch, root_item=self.root_item)
# Tags
for tag in task.get_tags():
self.domain.add_tag(tag)
elif self.domain.was_up():
self.domain.add_history(epoch, root_item=epoch)

58
bin/lib/crawlers.py
View file

@ -967,6 +967,7 @@ class CrawlerScheduler:
task_uuid = create_task(meta['url'], depth=meta['depth'], har=meta['har'], screenshot=meta['screenshot'],
header=meta['header'],
cookiejar=meta['cookiejar'], proxy=meta['proxy'],
tags=meta['tags'],
user_agent=meta['user_agent'], parent='scheduler', priority=40)
if task_uuid:
schedule.set_task(task_uuid)
@ -1069,6 +1070,14 @@ class CrawlerSchedule:
def _set_field(self, field, value):
return r_crawler.hset(f'schedule:{self.uuid}', field, value)
def get_tags(self):
return r_crawler.smembers(f'schedule:tags:{self.uuid}')
def set_tags(self, tags=[]):
for tag in tags:
r_crawler.sadd(f'schedule:tags:{self.uuid}', tag)
# Tag.create_custom_tag(tag)
def get_meta(self, ui=False):
meta = {
'uuid': self.uuid,
@ -1083,6 +1092,7 @@ class CrawlerSchedule:
'cookiejar': self.get_cookiejar(),
'header': self.get_header(),
'proxy': self.get_proxy(),
'tags': self.get_tags(),
}
status = self.get_status()
if ui:
@ -1098,6 +1108,7 @@ class CrawlerSchedule:
meta = {'uuid': self.uuid,
'url': self.get_url(),
'user': self.get_user(),
'tags': self.get_tags(),
'next_run': self.get_next_run(r_str=True)}
status = self.get_status()
if isinstance(status, ScheduleStatus):
@ -1106,7 +1117,7 @@ class CrawlerSchedule:
return meta
def create(self, frequency, user, url,
depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None):
depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]):
if self.exists():
raise Exception('Error: Monitor already exists')
@ -1135,6 +1146,9 @@ class CrawlerSchedule:
if user_agent:
self._set_field('user_agent', user_agent)
if tags:
self.set_tags(tags)
r_crawler.sadd('scheduler:schedules', self.uuid)
def delete(self):
@ -1148,12 +1162,13 @@ class CrawlerSchedule:
# delete meta
r_crawler.delete(f'schedule:{self.uuid}')
r_crawler.delete(f'schedule:tags:{self.uuid}')
r_crawler.srem('scheduler:schedules', self.uuid)
def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None):
def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]):
schedule_uuid = gen_uuid()
schedule = CrawlerSchedule(schedule_uuid)
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent)
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
return schedule_uuid
# TODO sanityze UUID
@ -1289,6 +1304,11 @@ def get_captures_status():
status.append(meta)
return status
def delete_captures():
for capture_uuid in get_crawler_captures():
capture = CrawlerCapture(capture_uuid)
capture.delete()
##-- CRAWLER STATE --##
@ -1371,6 +1391,14 @@ class CrawlerTask:
def _set_field(self, field, value):
return r_crawler.hset(f'crawler:task:{self.uuid}', field, value)
def get_tags(self):
return r_crawler.smembers(f'crawler:task:tags:{self.uuid}')
def set_tags(self, tags):
for tag in tags:
r_crawler.sadd(f'crawler:task:tags:{self.uuid}', tag)
# Tag.create_custom_tag(tag)
def get_meta(self):
meta = {
'uuid': self.uuid,
@ -1385,6 +1413,7 @@ class CrawlerTask:
'header': self.get_header(),
'proxy': self.get_proxy(),
'parent': self.get_parent(),
'tags': self.get_tags(),
}
return meta
@ -1392,7 +1421,7 @@ class CrawlerTask:
# TODO SANITIZE PRIORITY
# PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100
def create(self, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None,
user_agent=None, parent='manual', priority=0):
user_agent=None, tags=[], parent='manual', priority=0):
if self.exists():
raise Exception('Error: Task already exists')
@ -1423,7 +1452,7 @@ class CrawlerTask:
# TODO SANITIZE COOKIEJAR -> UUID
# Check if already in queue
hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header)
hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags)
if r_crawler.hexists(f'crawler:queue:hash', hash_query):
self.uuid = r_crawler.hget(f'crawler:queue:hash', hash_query)
return self.uuid
@ -1444,6 +1473,9 @@ class CrawlerTask:
if user_agent:
self._set_field('user_agent', user_agent)
if tags:
self.set_tags(tags)
r_crawler.hset('crawler:queue:hash', hash_query, self.uuid)
self._set_field('hash', hash_query)
r_crawler.zadd('crawler:queue', {self.uuid: priority})
@ -1483,10 +1515,10 @@ class CrawlerTask:
# TODO move to class ???
def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header):
def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags):
to_enqueue = {'domain': domain, 'depth': depth, 'har': har, 'screenshot': screenshot,
'priority': priority, 'proxy': proxy, 'cookiejar': cookiejar, 'user_agent': user_agent,
'header': header}
'header': header, 'tags': tags}
if priority != 0:
to_enqueue['url'] = url
return hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest()
@ -1502,7 +1534,7 @@ def add_task_to_lacus_queue():
# PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100
def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None,
user_agent=None, parent='manual', priority=0, task_uuid=None):
user_agent=None, tags=[], parent='manual', priority=0, task_uuid=None):
if task_uuid:
if CrawlerTask(task_uuid).exists():
task_uuid = gen_uuid()
@ -1510,7 +1542,7 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
task_uuid = gen_uuid()
task = CrawlerTask(task_uuid)
task_uuid = task.create(url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar,
proxy=proxy, user_agent=user_agent, parent=parent, priority=priority)
proxy=proxy, user_agent=user_agent, tags=tags, parent=parent, priority=priority)
return task_uuid
@ -1586,15 +1618,17 @@ def api_add_crawler_task(data, user_id=None):
if verify[1] != 200:
return verify
tags = data.get('tags', [])
if frequency:
# TODO verify user
return create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None,
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None), 200
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags), 200
else:
# TODO HEADERS
# TODO USER AGENT
return create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None,
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None,
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags,
parent='manual', priority=90), 200
@ -1870,6 +1904,8 @@ def test_ail_crawlers():
load_blacklist()
# if __name__ == '__main__':
# delete_captures()
# item_id = 'crawled/2023/02/20/data.gz'
# item = Item(item_id)
# content = item.get_content()

35
var/www/blueprints/crawler_splash.py
View file

@ -96,7 +96,8 @@ def manual():
is_manager_connected=crawlers.get_lacus_connection_metadata(),
crawlers_types=crawlers_types,
proxies=proxies,
l_cookiejar=l_cookiejar)
l_cookiejar=l_cookiejar,
tags_selector_data=Tag.get_tags_selector_data())
@crawler_splash.route("/crawlers/send_to_spider", methods=['POST'])
@ -113,6 +114,34 @@ def send_to_spider():
depth_limit = request.form.get('depth_limit')
cookiejar_uuid = request.form.get('cookiejar')
# TAGS
tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except:
galaxies_tags = []
else:
galaxies_tags = []
# custom tags
if tags:
tags = tags.split()
else:
tags = []
escaped = []
for tag in tags:
escaped.append(tag)
tags = escaped + taxonomies_tags + galaxies_tags
# Frequency
if request.form.get('crawler_scheduler'):
frequency = request.form.get('frequency')
@ -147,6 +176,8 @@ def send_to_spider():
data['proxy'] = proxy
if cookiejar_uuid:
data['cookiejar'] = cookiejar_uuid
if tags:
data['tags'] = tags
# print(data)
res = crawlers.api_add_crawler_task(data, user_id=user_id)
@ -163,6 +194,7 @@ def scheduler_dashboard():
# print(schedulers)
# TODO list currently queued ?
return render_template("crawler_scheduler_dashboard.html",
bootstrap_label=bootstrap_label,
schedulers=schedulers,
is_manager_connected=crawlers.get_lacus_connection_metadata())
@ -176,6 +208,7 @@ def schedule_show():
abort(404)
meta = schedule.get_meta(ui=True)
return render_template("crawler_schedule_uuid.html",
bootstrap_label=bootstrap_label,
meta=meta)
@crawler_splash.route("/crawlers/schedule/delete", methods=['GET'])

23
var/www/templates/crawler/crawler_splash/crawler_manual.html
View file

@ -8,14 +8,16 @@
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
@ -119,6 +121,21 @@
</div>
<div class="card bg-light my-4">
<div class="card-header bg-dark text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{%for tag in dict_tracker['tags']%}{{tag}} {%endfor%}"{%endif%}{%endif%}>
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>

8
var/www/templates/crawler/crawler_splash/crawler_schedule_uuid.html
View file

@ -72,6 +72,14 @@
{% endif %}
</td>
</tr>
<tr>
<th>Tags</th>
<td>
{%for tag in meta['tags']%}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{%endfor%}
</td>
</tr>
</tbody>
</table>

9
var/www/templates/crawler/crawler_splash/crawler_scheduler_dashboard.html
View file

@ -45,7 +45,14 @@
<tbody id="tbody_last_crawled">
{% for meta in schedulers %}
<tr>
<td><a href="{{ url_for('crawler_splash.schedule_show') }}?uuid={{ meta['uuid'] }}">{{ meta['url'] }}</a></td>
<td>
<a href="{{ url_for('crawler_splash.schedule_show') }}?uuid={{ meta['uuid'] }}">{{ meta['url'] }}</a>
<div>
{% for tag in meta['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{%endfor%}
</div>
</td>
<td>{{ meta['status'] }}</td>
<td>
{% if not meta['next_run'] %}