mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [importer] add url_extract importer
This commit is contained in:
parent
199df870be
commit
3d4131a193
1 changed files with 43 additions and 0 deletions
43
bin/import/ail_json_importer/Ail_feeder_url_extract.py
Executable file
43
bin/import/ail_json_importer/Ail_feeder_url_extract.py
Executable file
|
@ -0,0 +1,43 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
"""
|
||||||
|
The JSON Receiver Module
|
||||||
|
================
|
||||||
|
|
||||||
|
Recieve Json Items (example: Twitter feeder)
|
||||||
|
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
|
# import item_basic
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
|
||||||
|
from Default_json import Default_json
|
||||||
|
|
||||||
|
class Ail_feeder_twitter(Default_json):
|
||||||
|
"""Twitter Feeder functions"""
|
||||||
|
|
||||||
|
def __init__(self, name, json_item):
|
||||||
|
super().__init__(name, json_item)
|
||||||
|
|
||||||
|
def get_feeder_name(self):
|
||||||
|
return 'urlextract'
|
||||||
|
|
||||||
|
# define item id
|
||||||
|
def get_item_id(self):
|
||||||
|
# use twitter timestamp ?
|
||||||
|
item_date = datetime.date.today().strftime("%Y/%m/%d")
|
||||||
|
item_id = str(self.json_item['meta']['twitter:url-extracted'])
|
||||||
|
item_id = '{}{}.gz'.format(file_name_paste[:215], str(uuid.uuid4()))
|
||||||
|
return os.path.join('urlextract', item_date, item_id) + '.gz'
|
||||||
|
|
||||||
|
# # TODO:
|
||||||
|
def process_json_meta(self, process):
|
||||||
|
'''
|
||||||
|
Process JSON meta filed.
|
||||||
|
'''
|
||||||
|
return None
|
Loading…
Reference in a new issue