mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [Tags] add backend, daterange selector
This commit is contained in:
Terrtia
parent
65e6eae00d
commit
a9e86231ad
5 changed files with 128 additions and 16 deletions
13
bin/Tags.py
13
bin/Tags.py
|
|
@ -17,8 +17,8 @@ from pubsublogger import publisher
|
|||
from Helper import Process
|
||||
from packages import Paste
|
||||
|
||||
def get_paste_date(paste_filename):
|
||||
l_directory = paste_filename.split('/')
|
||||
def get_item_date(item_filename):
|
||||
l_directory = item_filename.split('/')
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
||||
def set_tag_metadata(tag, date):
|
||||
|
|
@ -75,14 +75,15 @@ if __name__ == '__main__':
|
|||
if res == 1:
|
||||
print("new tags added : {}".format(tag))
|
||||
# add the path to the tag set
|
||||
curr_date = datetime.date.today().strftime("%Y%m%d")
|
||||
res = server.sadd('{}:{}'.format(tag, curr_date), path)
|
||||
#curr_date = datetime.date.today().strftime("%Y%m%d")
|
||||
item_date = get_item_date(path)
|
||||
res = server.sadd('{}:{}'.format(tag, item_date), path)
|
||||
if res == 1:
|
||||
print("new paste: {}".format(path))
|
||||
print(" tagged: {}".format(tag))
|
||||
set_tag_metadata(tag, curr_date)
|
||||
set_tag_metadata(tag, item_date)
|
||||
server_metadata.sadd('tag:{}'.format(path), tag)
|
||||
|
||||
curr_date = datetime.date.today().strftime("%Y%m%d")
|
||||
server.hincrby('daily_tags:{}'.format(curr_date), tag, 1)
|
||||
server.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
||||
p.populate_set_out(message, 'MISP_The_Hive_feeder')
|
||||
|
|
|
|||
|
|
@ -154,6 +154,8 @@ if baseUrl != '':
|
|||
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
|
||||
max_tags_result = 50
|
||||
|
||||
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
|
||||
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ r_serv_statistics = Flask_config.r_serv_statistics
|
|||
max_preview_char = Flask_config.max_preview_char
|
||||
max_preview_modal = Flask_config.max_preview_modal
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
max_tags_result = Flask_config.max_tags_result
|
||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||
|
||||
Tags = Blueprint('Tags', __name__, template_folder='templates')
|
||||
|
|
@ -68,20 +69,61 @@ def get_tags_with_synonyms(tag):
|
|||
else:
|
||||
return {'name':tag,'id':tag}
|
||||
|
||||
def get_item_date(item_filename):
|
||||
l_directory = item_filename.split('/')
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
def get_all_dates_range(date_from, date_to):
|
||||
all_dates = {}
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
try:
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
except:
|
||||
pass
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
|
||||
date_to = date_from
|
||||
|
||||
else:
|
||||
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
||||
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
||||
all_dates['date_from'] = date_from
|
||||
all_dates['date_to'] = date_to
|
||||
all_dates['date_range'] = date_range
|
||||
return all_dates
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@Tags.route("/tags/", methods=['GET'])
|
||||
def Tags_page():
|
||||
current_date = datetime.date.today().strftime("%Y-%m-%d")
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
dates = get_all_dates_range(date_from, date_to)
|
||||
|
||||
tags = request.args.get('ltags')
|
||||
if tags is None:
|
||||
return render_template("Tags.html", date_from=current_date, date_to=current_date)
|
||||
|
||||
else:
|
||||
|
||||
|
||||
tags = request.args.get('ltags')
|
||||
|
||||
list_tags = tags.split(',')
|
||||
|
|
@ -97,11 +139,23 @@ def Tags_page():
|
|||
print('empty')
|
||||
# 1 tag
|
||||
elif len(list_tags) < 2:
|
||||
tagged_pastes = r_serv_tags.smembers(list_tags[0])
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
|
||||
|
||||
# 2 tags or more
|
||||
else:
|
||||
tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tag_keys = []
|
||||
for tag in list_tags:
|
||||
tag_keys.append('{}:{}'.format(tag, date))
|
||||
|
||||
if len(tag_keys) > 1:
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
|
||||
else:
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0])
|
||||
tagged_pastes.extend(daily_items)
|
||||
|
||||
else :
|
||||
return 'INCORRECT INPUT'
|
||||
|
|
@ -113,7 +167,21 @@ def Tags_page():
|
|||
allPastes = list(tagged_pastes)
|
||||
paste_tags = []
|
||||
|
||||
for path in allPastes[0:50]: ######################moduleName
|
||||
try:
|
||||
page = int(request.args.get('page'))
|
||||
except:
|
||||
page = 1
|
||||
if page <= 0:
|
||||
page = 1
|
||||
nb_page_max = len(tagged_pastes)/(max_tags_result)
|
||||
if not nb_page_max.is_integer():
|
||||
nb_page_max = int(nb_page_max)+1
|
||||
if page > nb_page_max:
|
||||
page = nb_page_max
|
||||
start = max_tags_result*(page -1)
|
||||
stop = max_tags_result*page
|
||||
|
||||
for path in allPastes[start:stop]: ######################moduleName
|
||||
all_path.append(path)
|
||||
paste = Paste.Paste(path)
|
||||
content = paste.get_p_content()
|
||||
|
|
@ -153,8 +221,9 @@ def Tags_page():
|
|||
all_path=all_path,
|
||||
tags=tags,
|
||||
list_tag = list_tag,
|
||||
date_from=current_date,
|
||||
date_to=current_date,
|
||||
date_from=dates['date_from'],
|
||||
date_to=dates['date_to'],
|
||||
page=page, nb_page_max=nb_page_max,
|
||||
paste_tags=paste_tags,
|
||||
bootstrap_label=bootstrap_label,
|
||||
content=all_content,
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@
|
|||
|
||||
<div>
|
||||
{%if all_path%}
|
||||
<table class="table table-bordered table-hover " id="myTable_">
|
||||
<table class="table table-bordered table-hover" id="myTable_">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Date</th>
|
||||
|
|
@ -142,6 +142,42 @@
|
|||
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="d-flex justify-content-center border-top mt-2">
|
||||
<nav class="mt-4" aria-label="...">
|
||||
<ul class="pagination">
|
||||
<li class="page-item {%if page==1%}disabled{%endif%}">
|
||||
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">Previous</a>
|
||||
</li>
|
||||
|
||||
{%if page>3%}
|
||||
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page=1&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">1</a></li>
|
||||
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
|
||||
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page-1}}</a></li>
|
||||
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page}}</a></li>
|
||||
{%else%}
|
||||
{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-2}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page-2}}</a></li>{%endif%}
|
||||
{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page-1}}</a></li>{%endif%}
|
||||
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page}}</a></li>
|
||||
{%endif%}
|
||||
|
||||
{%if nb_page_max-page>3%}
|
||||
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{page+1}}</a></li>
|
||||
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
|
||||
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{nb_page_max}}</a></li>
|
||||
{%else%}
|
||||
{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-2}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{nb_page_max-2}}</a></li>{%endif%}
|
||||
{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{nb_page_max-1}}</a></li>{%endif%}
|
||||
{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}">{{nb_page_max}}</a></li>{%endif%}
|
||||
{%endif%}
|
||||
|
||||
<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
|
||||
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}<ags={{tags}}" aria-disabled="true">Next</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
{%endif%}
|
||||
</div>
|
||||
|
||||
|
|
@ -247,7 +283,9 @@ $(document).ready(function(){
|
|||
<script>
|
||||
function searchTags() {
|
||||
var data = ltags.getValue();
|
||||
window.location.replace("{{ url_for('Tags.Tags_page') }}?ltags=" + data);
|
||||
var date_from = $('#date-range-from-input').val();
|
||||
var date_to =$('#date-range-to-input').val();
|
||||
window.location.replace("{{ url_for('Tags.Tags_page') }}?date_from="+date_from+"&date_to="+date_to+"<ags=" + data);
|
||||
}
|
||||
function emptyTags() {
|
||||
ltags.clear();
|
||||
|
|
|
|||
|
|
@ -241,6 +241,7 @@
|
|||
});
|
||||
</script>
|
||||
|
||||
{% if crawler_metadata['get_metadata'] %}
|
||||
<script>
|
||||
var ctx = canvas.getContext('2d'), img = new Image();
|
||||
|
||||
|
|
@ -287,6 +288,7 @@
|
|||
|
||||
blocks.addEventListener('change', pixelate, false);
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
<div id="container-show-more" class="text-center">
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue