AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [Update] add 1.5 update, remove absolute paths from DB

Browse source
This commit is contained in:
Terrtia 2018-11-21 17:07:29 +01:00
parent aaa277b8a0
commit 3272ffa714
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 160 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

160
update/v1.5/Update.py Executable file
View file

@ -0,0 +1,160 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import redis
import configparser
if __name__ == '__main__':
start_deb = time.time()
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
r_serv_metadata = redis.StrictRedis(
host=cfg.get("ARDB_Metadata", "host"),
port=cfg.getint("ARDB_Metadata", "port"),
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
r_serv_tag = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
r_serv_onion = redis.StrictRedis(
host=cfg.get("ARDB_Onion", "host"),
port=cfg.getint("ARDB_Onion", "port"),
db=cfg.getint("ARDB_Onion", "db"),
decode_responses=True)
## Update metadata ##
print('Updating ARDB_Metadata ...')
index = 0
start = time.time()
for key in r_serv_metadata.scan_iter('*'):
list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]:
for hash_key, value in list_data[1].items():
r_serv_metadata.hdel(key, hash_key)
if PASTES_FOLDER in hash_key:
new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
else:
new_hash = hash_key
if PASTES_FOLDER in value:
new_value = value.replace(PASTES_FOLDER, '', 1)
else:
new_value = value
index = index +1
r_serv_metadata.hset(key, new_hash, new_value)
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]:
for elem in list_data[1]:
zset_key = elem[0]
value = int(elem[1])
r_serv_metadata.zrem(key, zset_key)
new_key = zset_key.replace(PASTES_FOLDER, '', 1)
index = index +1
r_serv_metadata.zincrby(key, new_key, value)
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
if not 'dup:' in key:
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]:
for set_value in list_data[1]:
r_serv_metadata.srem(key, set_value)
r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
index = index + 1
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
if PASTES_FOLDER in key:
new_key = key.replace(PASTES_FOLDER, '', 1)
# a set with this key already exist
if r_serv_metadata.exists(new_key):
# save data
for new_key_value in r_serv_metadata.smembers(new_key):
r_serv_metadata.sadd(key, new_key_value)
r_serv_metadata.rename(key, new_key)
index = index + 1
end = time.time()
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
print()
print('Updating ARDB_Tags ...')
index = 0
start = time.time()
tags_list = r_serv_tag.smembers('list_tags')
for tag in tags_list:
res = False
list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_pastes[1]:
for paste in list_pastes[1]:
r_serv_tag.srem(tag, paste)
r_serv_tag.sadd(tag, paste.replace(PASTES_FOLDER, '', 1))
index = index + 1
list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
end = time.time()
print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start))
print()
print('Updating ARDB_Onion ...')
index = 0
start = time.time()
for key in r_serv_onion.scan_iter('*'):
if key != 'mess_onion':
list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]:
for hash_key, value in list_data[1].items():
r_serv_onion.hdel(key, hash_key)
if PASTES_FOLDER in hash_key:
new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
else:
new_hash = hash_key
if PASTES_FOLDER in value:
new_value = value.replace(PASTES_FOLDER, '', 1)
else:
new_value = value
index = index +1
r_serv_onion.hset(key, new_hash, new_value)
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]:
for set_value in list_data[1]:
r_serv_onion.srem(key, set_value)
r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
index = index + 1
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
end = time.time()
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
print()
print('Done in {} s'.format(end - start_deb))