chg: [importer] add url_extract importer

This commit is contained in:
Terrtia 2020-04-30 09:54:26 +02:00
parent 199df870be
commit 3d4131a193
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0

View file

@ -0,0 +1,43 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The JSON Receiver Module
================
Recieve Json Items (example: Twitter feeder)
"""
import os
import json
import sys
import datetime
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
# import item_basic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
from Default_json import Default_json
class Ail_feeder_twitter(Default_json):
"""Twitter Feeder functions"""
def __init__(self, name, json_item):
super().__init__(name, json_item)
def get_feeder_name(self):
return 'urlextract'
# define item id
def get_item_id(self):
# use twitter timestamp ?
item_date = datetime.date.today().strftime("%Y/%m/%d")
item_id = str(self.json_item['meta']['twitter:url-extracted'])
item_id = '{}{}.gz'.format(file_name_paste[:215], str(uuid.uuid4()))
return os.path.join('urlextract', item_date, item_id) + '.gz'
# # TODO:
def process_json_meta(self, process):
'''
Process JSON meta filed.
'''
return None