mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Publish the fetched onions on a ZMQ feed.
This commit is contained in:
parent
f15caa70f5
commit
8803c8447a
3 changed files with 8 additions and 2 deletions
|
@ -54,7 +54,8 @@ def fetch(p, r_cache, urls, domains, path):
|
|||
tempfile = process.stdout.read().strip()
|
||||
with open(tempfile, 'r') as f:
|
||||
filename = path + domain + '.gz'
|
||||
content = base64.standard_b64decode(f.read())
|
||||
fetched = f.read()
|
||||
content = base64.standard_b64decode(fetched)
|
||||
save_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "pastes"),
|
||||
filename)
|
||||
|
@ -65,6 +66,7 @@ def fetch(p, r_cache, urls, domains, path):
|
|||
ff.write(content)
|
||||
p.populate_set_out(save_path, 'Global')
|
||||
p.populate_set_out(url, 'ValidOnion')
|
||||
p.populate_set_out(fetched, 'FetchedOnion')
|
||||
yield url
|
||||
os.unlink(tempfile)
|
||||
else:
|
||||
|
|
|
@ -53,6 +53,10 @@ channel = 102
|
|||
address = tcp://127.0.0.1:5004
|
||||
channel = urls
|
||||
|
||||
[ZMQ_FetchedOnion]
|
||||
address = tcp://127.0.0.1:5005
|
||||
channel = FetchedOnion
|
||||
|
||||
[RedisPubSub]
|
||||
host = localhost
|
||||
port = 6381
|
||||
|
|
|
@ -37,7 +37,7 @@ subscribe = Redis_Mail
|
|||
|
||||
[Onion]
|
||||
subscribe = Redis_Onion
|
||||
publish = Redis_Global,Redis_ValidOnion
|
||||
publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion
|
||||
|
||||
[DumpValidOnion]
|
||||
subscribe = Redis_ValidOnion
|
||||
|
|
Loading…
Reference in a new issue