AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-27 00:07:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [urlexport importer] fix item name + redis config

Browse source
This commit is contained in:
Terrtia 2020-05-03 15:40:43 +02:00
parent b4cb14798c
commit 64fae5d7df
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
2 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bin/import/ail_json_importer/Ail_feeder_urlextract.py
View file

@ -33,11 +33,17 @@ class Ail_feeder_urlextract(Default_json):
# use twitter timestamp ?
item_date = datetime.date.today().strftime("%Y/%m/%d")
item_id = str(self.json_item['meta']['twitter:url-extracted'])
item_id = item_id.split('//')
if len(item_id) > 1:
item_id = ''.join(item_id[1:])
else:
item_id = item_id[0]
item_id = item_id.replace('/', '_')
if len(item_id) > 215:
item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4()))
else:
item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4()))
return os.path.join('urlextract', item_date, item_id) + '.gz'
return os.path.join('urlextract', item_date, item_id)
# # TODO:
def process_json_meta(self, process, item_id):

2
bin/lib/item_basic.py
View file

@ -11,6 +11,8 @@ config_loader = ConfigLoader.ConfigLoader()
# get and sanityze PASTE DIRECTORY
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def exist_item(item_id):