#!/usr/bin/env python3 # -*-coding:UTF-8 -* import os import sys import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import correlation import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader config_loader = ConfigLoader.ConfigLoader() serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None pgp = correlation.Correlation('pgpdump', ['key', 'mail', 'name']) def get_pgp(request_dict, pgp_type): # basic verification res = pgpdump.verify_correlation_field_request(request_dict, pgp_type) if res: return res # cerify address field_name = request_dict.get(pgp_type) return pgpdump.get_correlation(request_dict, pgp_type, field_name) def save_pgp_data(type_pgp, date, item_path, data): # create basic medata if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)): serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date) serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) else: last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen') if not last_seen: serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) else: if int(last_seen) < int(date): serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) # global set serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path) # daily serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1) # all type serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1) ## object_metadata # paste serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data) # domain object if Item.is_crawled(item_path): domain = Item.get_item_domain(item_path) serv_metadata.sadd('domain_pgpdump_{}:{}'.format(type_pgp, domain), data) serv_metadata.sadd('set_domain_pgpdump_{}:{}'.format(type_pgp, data), domain)