diff --git a/bin/import/ail_json_importer/Ail_feeder_twitter.py b/bin/import/ail_json_importer/Ail_feeder_twitter.py index 61ca4ec4..ec9fb0a2 100755 --- a/bin/import/ail_json_importer/Ail_feeder_twitter.py +++ b/bin/import/ail_json_importer/Ail_feeder_twitter.py @@ -12,8 +12,8 @@ import json import sys import datetime -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -# import item_basic +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import item_basic sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) from Default_json import Default_json @@ -39,4 +39,6 @@ class Ail_feeder_twitter(Default_json): ''' Process JSON meta filed. ''' + twitter_id = str(self.json_item['meta']['twitter:tweet_id']) + item_basic.add_map_obj_id_item_id(twitter_id, item_id, 'twitter_id') return None diff --git a/bin/import/ail_json_importer/Ail_feeder_urlextract.py b/bin/import/ail_json_importer/Ail_feeder_urlextract.py index 5c8e588d..bf72c714 100755 --- a/bin/import/ail_json_importer/Ail_feeder_urlextract.py +++ b/bin/import/ail_json_importer/Ail_feeder_urlextract.py @@ -53,9 +53,7 @@ class Ail_feeder_urlextract(Default_json): json_meta = self.get_json_meta() # # TODO: change me - parent_type = 'twitter' - item_parent = str(json_meta['parent:twitter:tweet_id']) - parent_date = datetime.date.today().strftime("%Y/%m/%d") - item_parent = os.path.join('twitter', parent_date, item_parent) + '.gz' + parent_type = 'twitter_id' - item_basic.add_item_parent(item_parent, item_id) + parent_id = str(json_meta['parent:twitter:tweet_id']) + item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 64a6d78c..97f85e96 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -26,10 +26,26 @@ def get_item_filepath(item_id): filename = os.path.join(PASTES_FOLDER, item_id) return os.path.realpath(filename) -def add_item_parent(item_parent, item_id): - if not exist_item(item_parent): +def add_item_parent_by_parent_id(parent_type, parent_id, item_id): + parent_item_id = get_obj_id_item_id(parent_type, parent_id) + if parent_item_id: + add_item_parent(item_parent, item_id) + +def add_item_parent(parent_item_id, item_id): + if not exist_item(parent_item_id): return False else: - r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', item_parent) - r_serv_metadata.sadd('paste_children:{}'.format(item_parent), item_id) + r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', parent_item_id) + r_serv_metadata.sadd('paste_children:{}'.format(parent_item_id), item_id) return True + +def add_map_obj_id_item_id(obj_id, item_id, obj_type): + if obj_type == 'twitter_id': + r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) + +def get_obj_id_item_id(parent_type, parent_id): + all_parents_type = ['twitter_id'] + if parent_type in all_parents_type: + return r_serv_metadata.hget('map:twitter_id:item_id', parent_id) + else: + return None diff --git a/installing_deps.sh b/installing_deps.sh index 93791280..49e7cb2e 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -109,7 +109,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh # init update version pushd ${AIL_HOME} # shallow clone -git fetch --tags --prune +git fetch --tags --prune --unshallow git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version echo "AIL current version:" git describe --abbrev=0 --tags