chg: [importer] add map twitter id - item id, add parents link between twitter and url extracted

This commit is contained in:
Terrtia 2020-05-05 11:24:33 +02:00
parent 1c412c9fed
commit 074ada504a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 28 additions and 12 deletions

View file

@ -12,8 +12,8 @@ import json
import sys
import datetime
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
# import item_basic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
from Default_json import Default_json
@ -39,4 +39,6 @@ class Ail_feeder_twitter(Default_json):
'''
Process JSON meta filed.
'''
twitter_id = str(self.json_item['meta']['twitter:tweet_id'])
item_basic.add_map_obj_id_item_id(twitter_id, item_id, 'twitter_id')
return None

View file

@ -53,9 +53,7 @@ class Ail_feeder_urlextract(Default_json):
json_meta = self.get_json_meta()
# # TODO: change me
parent_type = 'twitter'
item_parent = str(json_meta['parent:twitter:tweet_id'])
parent_date = datetime.date.today().strftime("%Y/%m/%d")
item_parent = os.path.join('twitter', parent_date, item_parent) + '.gz'
parent_type = 'twitter_id'
item_basic.add_item_parent(item_parent, item_id)
parent_id = str(json_meta['parent:twitter:tweet_id'])
item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)

View file

@ -26,10 +26,26 @@ def get_item_filepath(item_id):
filename = os.path.join(PASTES_FOLDER, item_id)
return os.path.realpath(filename)
def add_item_parent(item_parent, item_id):
if not exist_item(item_parent):
def add_item_parent_by_parent_id(parent_type, parent_id, item_id):
parent_item_id = get_obj_id_item_id(parent_type, parent_id)
if parent_item_id:
add_item_parent(item_parent, item_id)
def add_item_parent(parent_item_id, item_id):
if not exist_item(parent_item_id):
return False
else:
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', item_parent)
r_serv_metadata.sadd('paste_children:{}'.format(item_parent), item_id)
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', parent_item_id)
r_serv_metadata.sadd('paste_children:{}'.format(parent_item_id), item_id)
return True
def add_map_obj_id_item_id(obj_id, item_id, obj_type):
if obj_type == 'twitter_id':
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
def get_obj_id_item_id(parent_type, parent_id):
all_parents_type = ['twitter_id']
if parent_type in all_parents_type:
return r_serv_metadata.hget('map:twitter_id:item_id', parent_id)
else:
return None

View file

@ -109,7 +109,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
# init update version
pushd ${AIL_HOME}
# shallow clone
git fetch --tags --prune
git fetch --tags --prune --unshallow
git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
echo "AIL current version:"
git describe --abbrev=0 --tags