AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-26 15:57:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [Crawler] add domains blacklist

Browse source
This commit is contained in:
Terrtia 2018-09-28 16:29:09 +02:00
parent b3a6dc8487
commit 6328cc22b7
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 17 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bin/Crawler.py
View file

@ -130,6 +130,16 @@ if __name__ == '__main__':
db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True)
# load domains blacklist
try:
with open(os.environ['AIL_BIN']+'/torcrawler/blacklist_onion.txt', 'r') as f:
r_onion.delete('blacklist_{}'.format(type_hidden_service))
lines = f.read().splitlines()
for line in lines:
r_onion.sadd('blacklist_{}'.format(type_hidden_service), line)
except Exception:
pass
while True:
# Recovering the streamed message informations.
@ -160,7 +170,7 @@ if __name__ == '__main__':
print('domain: {}'.format(domain))
print('domain_url: {}'.format(domain_url))
if not r_onion.sismember('banned_{}'.format(type_hidden_service), domain):
if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain):
date = datetime.datetime.now().strftime("%Y%m%d")
date_month = datetime.datetime.now().strftime("%Y%m")

1
bin/torcrawler/blacklist_onion.txt Normal file
View file

@ -0,0 +1 @@
www.facebookcorewwwi.onion

1
var/www/modules/hiddenServices/Flask_hiddenServices.py
View file

@ -75,6 +75,7 @@ def hiddenServices_page():
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
for onion in last_onions:
metadata_onion = {}

4
var/www/modules/hiddenServices/templates/hiddenServices.html
View file

@ -120,6 +120,10 @@
<tr>
<td>Crawled Domains</td>
<td>{{ statDomains['total'] }}</td>
</tr>
<tr>
<td>Domains in Queue</td>
<td>{{ statDomains['domains_queue'] }}</td>
</tr>
</tbody>
</table>