From ddaad5db933f73ea29077f5532002285b9e66472 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 17 Apr 2019 17:07:09 +0200 Subject: [PATCH] chg: [v1.5 background update] create update stats --- bin/update-background.py | 7 ++- update/v1.4/Update-ARDB_Metadata.py | 38 +++++++++++- update/v1.4/Update-ARDB_Onions.py | 15 ++++- update/v1.4/Update-ARDB_Tags.py | 15 ++++- update/v1.4/Update-ARDB_Tags_background.py | 60 +++++++++++-------- var/www/modules/settings/Flask_settings.py | 9 +-- .../settings/templates/header_settings.html | 1 + .../settings/templates/settings_index.html | 8 +-- 8 files changed, 110 insertions(+), 43 deletions(-) create mode 100644 var/www/modules/settings/templates/header_settings.html diff --git a/bin/update-background.py b/bin/update-background.py index d6ea59b6..81441beb 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -47,12 +47,13 @@ if __name__ == "__main__": update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py') process = subprocess.run(['python' ,update_file]) - if if not r_serv.sismember('ail:update_v1.5', 'tags_background'): + if not r_serv.sismember('ail:update_v1.5', 'tags_background'): update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py') process = subprocess.run(['python' ,update_file]) if r_serv.scard('ail:update_v1.5') != 4: r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script') else: r_serv.delete('ail:update_in_progress') - r_serv_db.delete('ail:current_background_script') - r_serv.delete('ail:current_background_update', 'v1.5') + r_serv.delete('ail:current_background_script') + r_serv.delete('ail:current_background_script_stat') + r_serv.delete('ail:current_background_update') diff --git a/update/v1.4/Update-ARDB_Metadata.py b/update/v1.4/Update-ARDB_Metadata.py index eb6ae352..9d198f8d 100755 --- a/update/v1.4/Update-ARDB_Metadata.py +++ b/update/v1.4/Update-ARDB_Metadata.py @@ -97,13 +97,29 @@ if __name__ == '__main__': index = 0 start = time.time() + #update stats + r_serv.set('ail:current_background_script_stat', 0) + # Update base64 update_hash_item('base64') + + #update stats + r_serv.set('ail:current_background_script_stat', 20) # Update binary update_hash_item('binary') + + #update stats + r_serv.set('ail:current_background_script_stat', 40) # Update binary update_hash_item('hexadecimal') + #update stats + r_serv.set('ail:current_background_script_stat', 60) + + total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"') + nb_updated = 0 + last_progress = 0 + # Update onion metadata all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"') for item_path in all_crawled_items: @@ -131,17 +147,37 @@ if __name__ == '__main__': if PASTES_FOLDER in father: r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1)) - end = time.time() + nb_updated += 1 + progress = int((nb_updated * 30) /total_onion) + print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60)) + # update progress stats + if progress != last_progress: + r_serv.set('ail:current_background_script_stat', progress + 60) + last_progress = progress + + #update stats + r_serv.set('ail:current_background_script_stat', 90) ## update tracked term/set/regex # update tracked term update_tracked_terms('TrackedSetTermSet', 'tracked_{}') + + #update stats + r_serv.set('ail:current_background_script_stat', 93) # update tracked set update_tracked_terms('TrackedSetSet', 'set_{}') + + #update stats + r_serv.set('ail:current_background_script_stat', 96) # update tracked regex update_tracked_terms('TrackedRegexSet', 'regex_{}') + + #update stats + r_serv.set('ail:current_background_script_stat', 100) ## + end = time.time() + print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print() diff --git a/update/v1.4/Update-ARDB_Onions.py b/update/v1.4/Update-ARDB_Onions.py index f48fa0b9..cb143721 100755 --- a/update/v1.4/Update-ARDB_Onions.py +++ b/update/v1.4/Update-ARDB_Onions.py @@ -74,6 +74,7 @@ if __name__ == '__main__': decode_responses=True) r_serv.set('ail:current_background_script', 'onions') + r_serv.set('ail:current_background_script_stat', 0) ## Update Onion ## print('Updating ARDB_Onion ...') @@ -93,11 +94,15 @@ if __name__ == '__main__': all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1) if all_onion_history: for date_history in all_onion_history: - pass #print('onion_history:{}:{}'.format(onion_domain, date_history)) r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) r_serv_onion.delete('onion_history:{}'.format(onion_domain)) + #stats + total_domain = r_serv_onion.scard('full_onion_up') + nb_updated = 0 + last_progress = 0 + # clean up domain all_domain_up = r_serv_onion.smembers('full_onion_up') for onion_domain in all_domain_up: @@ -130,6 +135,14 @@ if __name__ == '__main__': r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80') r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen') + nb_updated += 1 + progress = int((nb_updated * 100) /total_domain) + print('{}/{} updated {}%'.format(nb_updated, total_domain, progress)) + # update progress stats + if progress != last_progress: + r_serv.set('ail:current_background_script_stat', progress) + last_progress = progress + end = time.time() print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) diff --git a/update/v1.4/Update-ARDB_Tags.py b/update/v1.4/Update-ARDB_Tags.py index 31d002d1..061a4b4d 100755 --- a/update/v1.4/Update-ARDB_Tags.py +++ b/update/v1.4/Update-ARDB_Tags.py @@ -58,11 +58,15 @@ if __name__ == '__main__': decode_responses=True) r_serv.set('ail:current_background_script', 'tags') + r_serv.set('ail:current_background_script_stat', 0) - if r_serv.exists('v1.5:onions') and r_serv.exists('v1.5:metadata'): + if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'): print('Updating ARDB_Tags ...') index = 0 + nb_tags_to_update = 0 + nb_updated = 0 + last_progress = 0 start = time.time() tags_list = r_serv_tag.smembers('list_tags') @@ -81,6 +85,7 @@ if __name__ == '__main__': tag_metadata[tag]['last_seen'] = 0 else: tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen']) + nb_tags_to_update += r_serv_tag.scard(tag) for tag in tags_list: @@ -123,6 +128,14 @@ if __name__ == '__main__': r_serv_tag.srem(tag, item_path) index = index + 1 + nb_updated += 1 + progress = int((nb_updated * 100) /nb_tags_to_update) + print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress)) + # update progress stats + if progress != last_progress: + r_serv.set('ail:current_background_script_stat', progress) + last_progress = progress + #flush browse importante pastes db r_important_paste_2018.flushdb() r_important_paste_2019.flushdb() diff --git a/update/v1.4/Update-ARDB_Tags_background.py b/update/v1.4/Update-ARDB_Tags_background.py index 1e117b65..fe1ec04b 100755 --- a/update/v1.4/Update-ARDB_Tags_background.py +++ b/update/v1.4/Update-ARDB_Tags_background.py @@ -47,35 +47,43 @@ if __name__ == '__main__': db=cfg.getint("ARDB_Tags", "db"), decode_responses=True) - r_serv.set('ail:current_background_script', 'tags_background') + if r_serv.sismember('ail:update_v1.5', 'tags'): - print('Updating ARDB_Tags ...') - start = time.time() + r_serv.set('ail:current_background_script', 'tags_background') + r_serv.set('ail:current_background_script_stat', 0) - #update item metadata tags - tag_not_updated = True - total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') - nb_updated = 0 - if total_to_update > 0: - while tag_not_updated: - item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename') - old_tag_item_key = 'tag:{}'.format(item_path) - new_item_path = item_path.replace(PASTES_FOLDER, '', 1) - new_tag_item_key = 'tag:{}'.format(new_item_path) - res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key) - if res == 0: - tags_key_fusion(old_tag_item_key, new_tag_item_key) - nb_updated += 1 - if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0: - tag_not_updated = False - else: - progress = int((nb_updated * 100) /total_to_update) - print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress)) - r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path) + print('Updating ARDB_Tags ...') + start = time.time() - end = time.time() + #update item metadata tags + tag_not_updated = True + total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') + nb_updated = 0 + last_progress = 0 + if total_to_update > 0: + while tag_not_updated: + item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename') + old_tag_item_key = 'tag:{}'.format(item_path) + new_item_path = item_path.replace(PASTES_FOLDER, '', 1) + new_tag_item_key = 'tag:{}'.format(new_item_path) + res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key) + if res == 0: + tags_key_fusion(old_tag_item_key, new_tag_item_key) + nb_updated += 1 + r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path) + if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0: + tag_not_updated = False + else: + progress = int((nb_updated * 100) /total_to_update) + print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress)) + # update progress stats + if progress != last_progress: + r_serv.set('ail:current_background_script_stat', progress) + last_progress = progress + + end = time.time() - print('Updating ARDB_Tags Done: {} s'.format(end - start)) + print('Updating ARDB_Tags Done: {} s'.format(end - start)) - r_serv.sadd('ail:update_v1.5', 'tags_background') + r_serv.sadd('ail:update_v1.5', 'tags_background') diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index 9d4d2d96..f81cf397 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -106,9 +106,6 @@ def get_background_update_stats_json(): else: update_stats['background_stats'] = int(update_stats['background_stats']) - ## DEBUG: - update_stats['background_stats'] =12 - update_progression = r_serv_db.scard('ail:update_{}'.format(current_update)) update_nb_scripts = dict_update_stat[current_update]['nb_background_update'] update_stats['update_stat'] = int(update_progression*100/update_nb_scripts) @@ -122,7 +119,11 @@ def get_background_update_stats_json(): else: update_stats['error_message'] = 'Please relaunch the bin/update-background.py script' else: - update_stats['error'] = False + if update_stats['background_name'] is None: + update_stats['error'] = True + update_stats['error_message'] = 'Please launch the bin/update-background.py script' + else: + update_stats['error'] = False return jsonify(update_stats) diff --git a/var/www/modules/settings/templates/header_settings.html b/var/www/modules/settings/templates/header_settings.html new file mode 100644 index 00000000..ab7220eb --- /dev/null +++ b/var/www/modules/settings/templates/header_settings.html @@ -0,0 +1 @@ +
  • Server Management
    • diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html index 8d4f0103..5b1e055a 100644 --- a/var/www/modules/settings/templates/settings_index.html +++ b/var/www/modules/settings/templates/settings_index.html @@ -58,7 +58,6 @@ {{git_metadata['current_branch']}} - {%if git_metadata['is_clone']%} - {%else%} - - This working directory is a fork - - {%endif%} @@ -196,7 +190,7 @@ update_progress(); //Interval var progress_interval = setInterval(function(){ update_progress() -}, 10000); +}, 4000);