ail-framework/bin/packages/Pgp.py

67 lines
2.2 KiB
Python
Executable file

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from Correlation import Correlation
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
class Pgp(Correlation):
def __init__(self):
super().__init__('pgpdump', ['key', 'mail', 'name'])
pgp = Pgp()
def get_pgp(request_dict, pgp_type):
# basic verification
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
if res:
return res
# cerify address
field_name = request_dict.get(pgp_type)
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
def save_pgp_data(type_pgp, date, item_path, data):
# create basic medata
if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else:
last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
if not last_seen:
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else:
if int(last_seen) < int(date):
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
# global set
serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
# daily
serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
# all type
serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
## object_metadata
# paste
serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
# domain object
if Item.is_crawled(item_path):
domain = Item.get_item_domain(item_path)
serv_metadata.sadd('domain_pgpdump_{}:{}'.format(type_pgp, domain), data)
serv_metadata.sadd('set_domain_pgpdump_{}:{}'.format(type_pgp, data), domain)