nicholas/showcase
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added blocklist.sh

Browse Source
This commit is contained in:
nicholasmanios
2020-03-18 18:56:15 -04:00
parent 192df8152f
commit 117733de4e
1 changed files with 108 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
aries/bind/blocklist.sh Normal file
View File

@@ -0,0 +1,108 @@
#!/bin/sh
## Lists are described here:
#https://firebog.net/
## Clear contents of existing blocklist & DNS config
truncate -s 0 /root/blocklists/raw_blocked_hosts.txt
truncate -s 0 /root/blocklists/raw_blocked_hosts_new.txt
truncate -s 0 /root/blocklists/blocked_hosts.txt
truncate -s 0 /root/blocklists/blocked_hosts_new.txt
truncate -s 0 /usr/local/etc/namedb/sinkhole.conf
## rules for below:
##
## remove commented lines
## grep -o '^[^#;]'
##
## remove 127.0.0.1
## sed -E '/^127\.0\.0\.1\s/d'
##
## remove 0.0.0.0
## sed -E '/^0\.0\.0\.0\s/d'
##
## remove HTML/XML tags
## sed -E 's/<[^>]*>//g'
##
## remove leading whitespace
## sed -E 's/^[ \t]*//'
##
## remove trailing whitespace
## sed -E 's/[ \t]*$//'
##
## remove blank/empty lines
## sed -E '/^[[:space:]]*$/d'
##
## removing the ^M char
## sed -E 's/^M//'
## first step, grab everything raw raw and dump into file (triple commented is DISABLED)
echo "Grabbing block lists from the internet..."
#curl -s curl -s https://raw.githubusercontent.com/StevenBlack/hosts/master/data/KADhosts/hosts >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://adaway.org/hosts.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://v.firebog.net/hosts/AdguardDNS.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://hosts-file.net/ad_servers.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://v.firebog.net/hosts/Easylist.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=domain;showintro=0" >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://raw.githubusercontent.com/StevenBlack/hosts/master/data/UncheckyAds/hosts >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://www.squidblacklist.org/downloads/dg-ads.acl >> /root/blocklists/raw_blocked_hosts_new.txt
###curl -s https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt >> /root/blocklists/raw_blocked_hosts_new.txt
###curl -s https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt >> /root/blocklists/raw_blocked_hosts_new.txt
#curl -s https://mirror1.malwaredomains.com/files/justdomains >> /root/blocklists/raw_blocked_hosts_new.txt
## second step, sanitize file
echo "Sanitizing blocklists..."
#grep -o '^[^#;]' /root/blocklists/raw_blocked_hosts_new.txt | sed -E '/^127\.0\.0\.1\s/d' | sed -E '/^0\.0\.0\.0\s/d' | sed -E 's/<[^>]*>//g' | sed -E 's/^[ \t]*//' | sed -E 's/[ \t]*$//' | sed -E '/^[[:space:]]*$/d' >> /root/blocklists/blocked_hosts_new.txt
## Download lists (-s silent output)
##curl -s https://www.dshield.org/feeds/suspiciousdomains_Medium.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\ )(.*\..*)$/\3/' | grep -v '^127.0.0.1\|0.0.0.0\|^::' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://raw.githubusercontent.com/StevenBlack/hosts/master/data/KADhosts/hosts | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(0\.0\.0\.0)(\ )(.*\..*)$/\3/' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
# this first record (adaway) is the only active one
curl -s https://adaway.org/hosts.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\ )(.*\..*)$/\3/' | grep -v '^127.0.0.1\|0.0.0.0\|^::' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://v.firebog.net/hosts/AdguardDNS.txt | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(0\.0\.0\.0)(\ )(.*\..*)$/\3/' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
##curl -s https://hosts-file.net/ad_servers.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\t)(.*\..*)$/\3/' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
##curl -s https://v.firebog.net/hosts/Easylist.txt | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
##curl -s "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=domain;showintro=0" | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://raw.githubusercontent.com/StevenBlack/hosts/master/data/UncheckyAds/hosts | sed -E 's/^(0\.0\.0\.0)(\ )(.*\..*)$/\3/' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
##curl -s https://www.squidblacklist.org/downloads/dg-ads.acl | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/[^[:print:]\t]//g' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\ )(.*\..*)$/\3/' | grep -v '^127.0.0.1\|0.0.0.0\|^::' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\ )(.*\..*)$/\3/' | grep -v '^127.0.0.1\|0.0.0.0\|^::' >> /root/blocklists/raw_blocked_hosts.txt
#curl -s https://mirror1.malwaredomains.com/files/justdomains | grep -v '^$\|^\s*\#' | grep -o '^[^#]*' | sed -E 's/^(127\.0\.0\.1)(\ )(.*\..*)$/\3/' | grep -v '^127.0.0.1\|0.0.0.0\|^::' >> /root/blocklists/raw_blocked_hosts.txt
## Remove duplicate lines
#awk '!seen[$0]++' /root/blocklists/raw_blocked_hosts.txt | awk '!seen[$0]++' > /root/blocklists/blocked_hosts.txt
awk '!_[$1]++' /root/blocklists/raw_blocked_hosts.txt > /root/blocklists/blocked_hosts.txt
## Third step, Create DNS zones for each domain in lists
echo "Creating DNS zones..."
while read line
do
echo "zone \"$line\" {type master; file \"/usr/local/etc/namedb/zones/db.sinkhole\"; check-names ignore;};" >> /usr/local/etc/namedb/sinkhole.conf
done < /root/blocklists/blocked_hosts.txt
#done < /root/blocklists/blocked_hosts_new.txt
## Fourth step, reload namedb with new zones
echo "Reloading named..."
rndc reconfig
rndc reload