Publish the fetched onions on a ZMQ feed.

bin/Onion.py

@@ -54,7 +54,8 @@ def fetch(p, r_cache, urls, domains, path):
             tempfile = process.stdout.read().strip()
             with open(tempfile, 'r') as f:
                 filename = path + domain + '.gz'
-                content = base64.standard_b64decode(f.read())
+                fetched = f.read()
+                content = base64.standard_b64decode(fetched)
                 save_path = os.path.join(os.environ['AIL_HOME'],
                                          p.config.get("Directories", "pastes"),
                                          filename)
@@ -65,6 +66,7 @@ def fetch(p, r_cache, urls, domains, path):
                     ff.write(content)
                 p.populate_set_out(save_path, 'Global')
                 p.populate_set_out(url, 'ValidOnion')
+                p.populate_set_out(fetched, 'FetchedOnion')
                 yield url
             os.unlink(tempfile)
         else:

bin/packages/config.cfg.sample

@@ -53,6 +53,10 @@ channel = 102
 address = tcp://127.0.0.1:5004
 channel = urls
 
+[ZMQ_FetchedOnion]
+address = tcp://127.0.0.1:5005
+channel = FetchedOnion
+
 [RedisPubSub]
 host = localhost
 port = 6381

bin/packages/modules.cfg

@@ -37,7 +37,7 @@ subscribe = Redis_Mail
 
 [Onion]
 subscribe = Redis_Onion
-publish = Redis_Global,Redis_ValidOnion
+publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion
 
 [DumpValidOnion]
 subscribe = Redis_ValidOnion