mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
chg: [v1.5 background update] create update stats
This commit is contained in:
parent
4bff603d4f
commit
ddaad5db93
8 changed files with 110 additions and 43 deletions
|
@ -47,12 +47,13 @@ if __name__ == "__main__":
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
|
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
|
||||||
process = subprocess.run(['python' ,update_file])
|
process = subprocess.run(['python' ,update_file])
|
||||||
|
|
||||||
if if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
|
if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
|
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
|
||||||
process = subprocess.run(['python' ,update_file])
|
process = subprocess.run(['python' ,update_file])
|
||||||
if r_serv.scard('ail:update_v1.5') != 4:
|
if r_serv.scard('ail:update_v1.5') != 4:
|
||||||
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
|
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
|
||||||
else:
|
else:
|
||||||
r_serv.delete('ail:update_in_progress')
|
r_serv.delete('ail:update_in_progress')
|
||||||
r_serv_db.delete('ail:current_background_script')
|
r_serv.delete('ail:current_background_script')
|
||||||
r_serv.delete('ail:current_background_update', 'v1.5')
|
r_serv.delete('ail:current_background_script_stat')
|
||||||
|
r_serv.delete('ail:current_background_update')
|
||||||
|
|
|
@ -97,13 +97,29 @@ if __name__ == '__main__':
|
||||||
index = 0
|
index = 0
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 0)
|
||||||
|
|
||||||
# Update base64
|
# Update base64
|
||||||
update_hash_item('base64')
|
update_hash_item('base64')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 20)
|
||||||
# Update binary
|
# Update binary
|
||||||
update_hash_item('binary')
|
update_hash_item('binary')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 40)
|
||||||
# Update binary
|
# Update binary
|
||||||
update_hash_item('hexadecimal')
|
update_hash_item('hexadecimal')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 60)
|
||||||
|
|
||||||
|
total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"')
|
||||||
|
nb_updated = 0
|
||||||
|
last_progress = 0
|
||||||
|
|
||||||
# Update onion metadata
|
# Update onion metadata
|
||||||
all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
|
all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
|
||||||
for item_path in all_crawled_items:
|
for item_path in all_crawled_items:
|
||||||
|
@ -131,17 +147,37 @@ if __name__ == '__main__':
|
||||||
if PASTES_FOLDER in father:
|
if PASTES_FOLDER in father:
|
||||||
r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
|
r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
|
||||||
|
|
||||||
end = time.time()
|
nb_updated += 1
|
||||||
|
progress = int((nb_updated * 30) /total_onion)
|
||||||
|
print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60))
|
||||||
|
# update progress stats
|
||||||
|
if progress != last_progress:
|
||||||
|
r_serv.set('ail:current_background_script_stat', progress + 60)
|
||||||
|
last_progress = progress
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 90)
|
||||||
|
|
||||||
## update tracked term/set/regex
|
## update tracked term/set/regex
|
||||||
# update tracked term
|
# update tracked term
|
||||||
update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
|
update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 93)
|
||||||
# update tracked set
|
# update tracked set
|
||||||
update_tracked_terms('TrackedSetSet', 'set_{}')
|
update_tracked_terms('TrackedSetSet', 'set_{}')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 96)
|
||||||
# update tracked regex
|
# update tracked regex
|
||||||
update_tracked_terms('TrackedRegexSet', 'regex_{}')
|
update_tracked_terms('TrackedRegexSet', 'regex_{}')
|
||||||
|
|
||||||
|
#update stats
|
||||||
|
r_serv.set('ail:current_background_script_stat', 100)
|
||||||
##
|
##
|
||||||
|
|
||||||
|
end = time.time()
|
||||||
|
|
||||||
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
|
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
|
@ -74,6 +74,7 @@ if __name__ == '__main__':
|
||||||
decode_responses=True)
|
decode_responses=True)
|
||||||
|
|
||||||
r_serv.set('ail:current_background_script', 'onions')
|
r_serv.set('ail:current_background_script', 'onions')
|
||||||
|
r_serv.set('ail:current_background_script_stat', 0)
|
||||||
|
|
||||||
## Update Onion ##
|
## Update Onion ##
|
||||||
print('Updating ARDB_Onion ...')
|
print('Updating ARDB_Onion ...')
|
||||||
|
@ -93,11 +94,15 @@ if __name__ == '__main__':
|
||||||
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
|
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
|
||||||
if all_onion_history:
|
if all_onion_history:
|
||||||
for date_history in all_onion_history:
|
for date_history in all_onion_history:
|
||||||
pass
|
|
||||||
#print('onion_history:{}:{}'.format(onion_domain, date_history))
|
#print('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||||
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
|
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||||
r_serv_onion.delete('onion_history:{}'.format(onion_domain))
|
r_serv_onion.delete('onion_history:{}'.format(onion_domain))
|
||||||
|
|
||||||
|
#stats
|
||||||
|
total_domain = r_serv_onion.scard('full_onion_up')
|
||||||
|
nb_updated = 0
|
||||||
|
last_progress = 0
|
||||||
|
|
||||||
# clean up domain
|
# clean up domain
|
||||||
all_domain_up = r_serv_onion.smembers('full_onion_up')
|
all_domain_up = r_serv_onion.smembers('full_onion_up')
|
||||||
for onion_domain in all_domain_up:
|
for onion_domain in all_domain_up:
|
||||||
|
@ -130,6 +135,14 @@ if __name__ == '__main__':
|
||||||
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
|
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
|
||||||
r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
|
r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
|
||||||
|
|
||||||
|
nb_updated += 1
|
||||||
|
progress = int((nb_updated * 100) /total_domain)
|
||||||
|
print('{}/{} updated {}%'.format(nb_updated, total_domain, progress))
|
||||||
|
# update progress stats
|
||||||
|
if progress != last_progress:
|
||||||
|
r_serv.set('ail:current_background_script_stat', progress)
|
||||||
|
last_progress = progress
|
||||||
|
|
||||||
|
|
||||||
end = time.time()
|
end = time.time()
|
||||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||||
|
|
|
@ -58,11 +58,15 @@ if __name__ == '__main__':
|
||||||
decode_responses=True)
|
decode_responses=True)
|
||||||
|
|
||||||
r_serv.set('ail:current_background_script', 'tags')
|
r_serv.set('ail:current_background_script', 'tags')
|
||||||
|
r_serv.set('ail:current_background_script_stat', 0)
|
||||||
|
|
||||||
if r_serv.exists('v1.5:onions') and r_serv.exists('v1.5:metadata'):
|
if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'):
|
||||||
|
|
||||||
print('Updating ARDB_Tags ...')
|
print('Updating ARDB_Tags ...')
|
||||||
index = 0
|
index = 0
|
||||||
|
nb_tags_to_update = 0
|
||||||
|
nb_updated = 0
|
||||||
|
last_progress = 0
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
tags_list = r_serv_tag.smembers('list_tags')
|
tags_list = r_serv_tag.smembers('list_tags')
|
||||||
|
@ -81,6 +85,7 @@ if __name__ == '__main__':
|
||||||
tag_metadata[tag]['last_seen'] = 0
|
tag_metadata[tag]['last_seen'] = 0
|
||||||
else:
|
else:
|
||||||
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
|
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
|
||||||
|
nb_tags_to_update += r_serv_tag.scard(tag)
|
||||||
|
|
||||||
for tag in tags_list:
|
for tag in tags_list:
|
||||||
|
|
||||||
|
@ -123,6 +128,14 @@ if __name__ == '__main__':
|
||||||
r_serv_tag.srem(tag, item_path)
|
r_serv_tag.srem(tag, item_path)
|
||||||
index = index + 1
|
index = index + 1
|
||||||
|
|
||||||
|
nb_updated += 1
|
||||||
|
progress = int((nb_updated * 100) /nb_tags_to_update)
|
||||||
|
print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress))
|
||||||
|
# update progress stats
|
||||||
|
if progress != last_progress:
|
||||||
|
r_serv.set('ail:current_background_script_stat', progress)
|
||||||
|
last_progress = progress
|
||||||
|
|
||||||
#flush browse importante pastes db
|
#flush browse importante pastes db
|
||||||
r_important_paste_2018.flushdb()
|
r_important_paste_2018.flushdb()
|
||||||
r_important_paste_2019.flushdb()
|
r_important_paste_2019.flushdb()
|
||||||
|
|
|
@ -47,35 +47,43 @@ if __name__ == '__main__':
|
||||||
db=cfg.getint("ARDB_Tags", "db"),
|
db=cfg.getint("ARDB_Tags", "db"),
|
||||||
decode_responses=True)
|
decode_responses=True)
|
||||||
|
|
||||||
r_serv.set('ail:current_background_script', 'tags_background')
|
if r_serv.sismember('ail:update_v1.5', 'tags'):
|
||||||
|
|
||||||
print('Updating ARDB_Tags ...')
|
r_serv.set('ail:current_background_script', 'tags_background')
|
||||||
start = time.time()
|
r_serv.set('ail:current_background_script_stat', 0)
|
||||||
|
|
||||||
#update item metadata tags
|
print('Updating ARDB_Tags ...')
|
||||||
tag_not_updated = True
|
start = time.time()
|
||||||
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
|
|
||||||
nb_updated = 0
|
|
||||||
if total_to_update > 0:
|
|
||||||
while tag_not_updated:
|
|
||||||
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
|
|
||||||
old_tag_item_key = 'tag:{}'.format(item_path)
|
|
||||||
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
|
||||||
new_tag_item_key = 'tag:{}'.format(new_item_path)
|
|
||||||
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
|
|
||||||
if res == 0:
|
|
||||||
tags_key_fusion(old_tag_item_key, new_tag_item_key)
|
|
||||||
nb_updated += 1
|
|
||||||
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
|
|
||||||
tag_not_updated = False
|
|
||||||
else:
|
|
||||||
progress = int((nb_updated * 100) /total_to_update)
|
|
||||||
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
|
|
||||||
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
|
|
||||||
|
|
||||||
end = time.time()
|
#update item metadata tags
|
||||||
|
tag_not_updated = True
|
||||||
|
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
|
||||||
|
nb_updated = 0
|
||||||
|
last_progress = 0
|
||||||
|
if total_to_update > 0:
|
||||||
|
while tag_not_updated:
|
||||||
|
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
|
||||||
|
old_tag_item_key = 'tag:{}'.format(item_path)
|
||||||
|
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||||
|
new_tag_item_key = 'tag:{}'.format(new_item_path)
|
||||||
|
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
|
||||||
|
if res == 0:
|
||||||
|
tags_key_fusion(old_tag_item_key, new_tag_item_key)
|
||||||
|
nb_updated += 1
|
||||||
|
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
|
||||||
|
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
|
||||||
|
tag_not_updated = False
|
||||||
|
else:
|
||||||
|
progress = int((nb_updated * 100) /total_to_update)
|
||||||
|
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
|
||||||
|
# update progress stats
|
||||||
|
if progress != last_progress:
|
||||||
|
r_serv.set('ail:current_background_script_stat', progress)
|
||||||
|
last_progress = progress
|
||||||
|
|
||||||
|
end = time.time()
|
||||||
|
|
||||||
|
|
||||||
print('Updating ARDB_Tags Done: {} s'.format(end - start))
|
print('Updating ARDB_Tags Done: {} s'.format(end - start))
|
||||||
|
|
||||||
r_serv.sadd('ail:update_v1.5', 'tags_background')
|
r_serv.sadd('ail:update_v1.5', 'tags_background')
|
||||||
|
|
|
@ -106,9 +106,6 @@ def get_background_update_stats_json():
|
||||||
else:
|
else:
|
||||||
update_stats['background_stats'] = int(update_stats['background_stats'])
|
update_stats['background_stats'] = int(update_stats['background_stats'])
|
||||||
|
|
||||||
## DEBUG:
|
|
||||||
update_stats['background_stats'] =12
|
|
||||||
|
|
||||||
update_progression = r_serv_db.scard('ail:update_{}'.format(current_update))
|
update_progression = r_serv_db.scard('ail:update_{}'.format(current_update))
|
||||||
update_nb_scripts = dict_update_stat[current_update]['nb_background_update']
|
update_nb_scripts = dict_update_stat[current_update]['nb_background_update']
|
||||||
update_stats['update_stat'] = int(update_progression*100/update_nb_scripts)
|
update_stats['update_stat'] = int(update_progression*100/update_nb_scripts)
|
||||||
|
@ -122,7 +119,11 @@ def get_background_update_stats_json():
|
||||||
else:
|
else:
|
||||||
update_stats['error_message'] = 'Please relaunch the bin/update-background.py script'
|
update_stats['error_message'] = 'Please relaunch the bin/update-background.py script'
|
||||||
else:
|
else:
|
||||||
update_stats['error'] = False
|
if update_stats['background_name'] is None:
|
||||||
|
update_stats['error'] = True
|
||||||
|
update_stats['error_message'] = 'Please launch the bin/update-background.py script'
|
||||||
|
else:
|
||||||
|
update_stats['error'] = False
|
||||||
|
|
||||||
return jsonify(update_stats)
|
return jsonify(update_stats)
|
||||||
|
|
||||||
|
|
1
var/www/modules/settings/templates/header_settings.html
Normal file
1
var/www/modules/settings/templates/header_settings.html
Normal file
|
@ -0,0 +1 @@
|
||||||
|
<li id='page-hiddenServices'><a href="{{ url_for('settings.settings_page') }}"><i class="fa fa-cog"></i> Server Management </a></li>
|
|
@ -58,7 +58,6 @@
|
||||||
{{git_metadata['current_branch']}}
|
{{git_metadata['current_branch']}}
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{%if git_metadata['is_clone']%}
|
|
||||||
<tr
|
<tr
|
||||||
{%if git_metadata['new_git_update_available']%}
|
{%if git_metadata['new_git_update_available']%}
|
||||||
class="table-warning"
|
class="table-warning"
|
||||||
|
@ -85,11 +84,6 @@
|
||||||
{{git_metadata['last_local_tag']}}
|
{{git_metadata['last_local_tag']}}
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{%else%}
|
|
||||||
<tr>
|
|
||||||
<td>This working directory is a fork</td>
|
|
||||||
</tr>
|
|
||||||
{%endif%}
|
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
@ -196,7 +190,7 @@ update_progress();
|
||||||
//Interval
|
//Interval
|
||||||
var progress_interval = setInterval(function(){
|
var progress_interval = setInterval(function(){
|
||||||
update_progress()
|
update_progress()
|
||||||
}, 10000);
|
}, 4000);
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
</html>
|
</html>
|
||||||
|
|
Loading…
Reference in a new issue