[SOLVED] Checking IP's against a whole lot of DNSBL's

frater · 12-17-2010, 02:53 PM

@work we use Zabbix and also IP-audit for monitoring.
Each day we have a list of outgoing SMTP-servers on our IPaudit server.

This script will take that list and check them all against a whole bunch of DNSBL's

# cat /usr/local/sbin/check_rbl

Code:

#!/bin/bash
#####################################################
# check_rbl
#####################################################
# 17-12-2010 by JP van Melis
#
# Use at your own risk!
#####################################################
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

# If you don't want to use your own DNS-server
# Use Google's, Blank it if you want to use the default always.
DNServer=8.8.8.8
URL=http://blog.penumbra.be/files/zabbix/rbl_list.txt
dnsbl=/etc/dnsbl.txt
IPAUDITDIR=/home/ipaudit/reports/daily/server2/html

DNSBL=
TESTDNSBL=0
LISTIPS=0
ZABBIX_SENDER=0
ZABBIX_CONF=/etc/zabbix/zabbix_agentd.conf

TouchedToday=
Time=2
DEBUG=0

NAME="$0"
[ -z "`which readlink`" ] || NAME="`readlink -f "${NAME}"`"
BASENAME=`basename "${NAME}"`
reportfile=/var/log/${BASENAME}.report

while getopts zltr:d: name
do
  case $name in
    z)   ZABBIX_SENDER=1;;
    l)   LISTIPS=1;;
    t)   TESTDNSBL=1;;
    r)   reportfile="$OPTARG";;
    d)   DNSBL="$OPTARG";;
    ?)   printf "Usage: %s: [-d] [-l] [-t] [-r <reportfile>] [(<IP>|<domain>|<file>)]\n" $0
    exit 2;;
  esac
done
shift $(($OPTIND - 1))

_exit ()
{
  rm -f $ftmp1 2>/dev/null
  rm -f $ftmp2 2>/dev/null
  rm -f $ftmp3 2>/dev/null
  exit $1
}

function zsend {
  key="dnsbl[$1]"
  if  [ ${DEBUG} -ne 0 ] ; then
    echo "Send key \"${key}\" with value \"${2}\"" >&2
    zabbix_sender -vv -c $ZABBIX_CONF -k "${key}" -o "${2}"
  else
    zabbix_sender -c $ZABBIX_CONF -k "${key}" -o "${2}" 2>&1 >/dev/null
  fi
}

if [ ${ZABBIX_SENDER} -eq 1 ] ; then
  [ -e ${ZABBIX_CONF} ] || ZABBIX_CONF=/opt${ZABCONF}
  if [ ! -e ${ZABBIX_CONF} ] ; then
    echo "Unable to find ${ZABBIX_CONF}, can't continue" >&2
    exit 1
  fi
fi

ftmp1=`mktemp`
ftmp2=`mktemp`
ftmp3=`mktemp`

if [ ${LISTIPS} -eq 0 ] ; then

  # No DNSBL given... use standard file or create one by recursively calling myself
  [ -z "${DNSBL}" ] && [ -e ${dnsbl} ] && [ ! -d ${dnsbl} ] && DNSBL="${dnsbl}"
  if [ -z "${DNSBL}" ] ; then
    echo -e "No ${dnsbl} found...\nI will make an attempt to create one by downloading ${URL}!"
    if wget -O $ftmp3 ${URL} 2>/dev/null ; then
      lines1=`cat $ftmp3 | wc -l`
      # make it look older, so it will get checked next run
      touch -d "last week" $ftmp3
      echo "Downloaded ${lines1} DNSBL's, I will now recursively call myself to clean the file up!"
      $0 -d $ftmp3 -t >${dnsbl}
      lines2=`cat ${dnsbl} | wc -l`
      if [ $lines2 -eq $lines1 ] ; then
        echo "All DNSBL's seem to be valid!"
      else
        echo -e "Only $lines2 survived the test!\n\nThese were found to be invalid:\n"
        while read failed ; do
          grep -q "${failed}" ${dnsbl} || echo "${failed}" | awk '{print $1}' | sed 's/\.$//' | sed "s/.*/host -tA -W$Time 2.0.0.127.&. ; host -tA -W$Time 1.0.0.127.&./"
        done <$ftmp3
      fi
      echo -e "\nThe default DNSBL list (${dnsbl}) is created, you should not see this anymore"
      _exit 1
    else
      echo "Download from ${URL} failed"
      _exit 1
    fi
  elif echo "${DNSBL}" | grep -q '/' ; then
    # Create list from file
    if [ -e "${DNSBL}" ] && [ ! -d "${DNSBL}" ] ; then
      if head -n1 "${DNSBL}" | grep -q '^[-0-9A-Za-z_.]* [01]$' ; then
        cp -p "${DNSBL}" $ftmp2
        dirname=`dirname ${DNSBL}`
        basename=`basename ${DNSBL}`
        TouchedToday="`find "${dirname}" -maxdepth 1 -name "${basename}"  -mtime 0`"
      else
        cat "${DNSBL}" | awk '{print $1}' | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
      fi
    fi
  else
    # Create list from command-line parameter
    echo "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
  fi

  # if file is unchecked or its first line doesn't end with ' 0' oe ' 1'
  if [ -z "${TouchedToday}" ] ; then
    # Check if DNSBL-server is working
    while read blacklistline ; do
      # remove a trailing dot, so we can add one without making a duplicate
      blacklist=`echo "${blacklistline}" | awk '{print $1}' | sed 's/\.$//'`
      if host -tA -W$Time 2.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.' ; then
        host -tA -W$Time 1.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 1" >>${ftmp3}
      elif [ ! -z "${DNServer}" ] ; then
        if host -tA -W$Time 2.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.' ; then
          host -tA -W$Time 1.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 0" >>${ftmp3}
        fi
      fi
    done<${ftmp2}
  else
    cp -p $ftmp2 $ftmp3
  fi

  if [ ! -s ${ftmp3} ] ; then
    echo "No valid DNSBL-servers found"
    _exit 1
  elif echo "${DNSBL}" | grep -q '/' ; then
    if [ -z "${TouchedToday}" ] ; then
      # If a file is used for the DNSBL's then check if all are valid..
      # If so, then 'touch it', so it doesn't need checking today.
      lines1=`cat "${DNSBL}" 2>/dev/null | wc -l`
      lines2=`cat "${ftmp3}" 2>/dev/null | wc -l`
      [ $lines1 -eq $lines2 ] && touch "${DNSBL}"
    fi
  fi

  if [ $TESTDNSBL -eq 1 ] ; then
    sort -u ${ftmp3}
    _exit 0
  fi

  [ -s ${ftmp3} ] || exit 1
fi

# delete surrounding spaces
IPSTOTEST="`echo "$*" | sed 's/^ *//;s/ *$//'`"

if [ -z "${IPSTOTEST}" ] ; then
  if [ -d "${IPAUDITDIR}" ] ; then
    # check IPaudit's daily list
    fname=`ls -1t ${IPAUDITDIR}/*.html 2>/dev/null | head -n1`
    [ -e "${fname}" ] && grep -o 'remote_port=25&ip=[0-9.]*' "${fname}" |  awk -F= '{print $3}' | sed 's/^0*//;s/\.0*/./g' >$ftmp1
  else
    wget -O - http://checkip.dyndns.org/ 2>/dev/null | grep -io 'IP Address: [0-9.]*' | awk '{print $3}' >$ftmp1
  fi
elif echo "${IPSTOTEST}" | grep -q '/' ; then
  [ -e "${IPSTOTEST}" ] && grep -o '[-0-9A-Za-z_.]*' "${IPSTOTEST}" | sed 's/\.$//' >$ftmp1
else
  echo "${IPSTOTEST}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp1
fi

# Convert host names to IP's if it's not done already.
echo -n '' >$ftmp2
while read IP ; do
  if echo "${IP}" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*$' ; then
    echo "${IP}" >>$ftmp2
  else
    ResolvedIP="`host -tA "${IP}." | grep -o 'has address [0-9.]*' | awk '{printf "%s ", $3}'`"
    [ -z "${ResolvedIP}" ] || echo "${ResolvedIP}${IP}." >>$ftmp2
  fi
done <$ftmp1

if [ ! -s ${ftmp2} ] ; then
  echo "NO IP's found" 2>/dev/null
  _exit 1
fi

if [ ${LISTIPS} -eq 1 ] ; then
  cat ${ftmp2}
  _exit 0
fi

someonelisted=0
echo -n '' >${reportfile}
while read IPline ; do

  echo -n '' >$ftmp1
  # extract Domain from IPline and cut that Domain out of IPline
  Domain="`echo "${IPline}" | sed -e 's/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*//g' | sed 's/ *//g'`"
  [ -z "${Domain}" ] || IPline="`echo "${IPline}" | sed -e "s/${Domain}//"`"
  for IP in ${IPline} ; do

    oct1=$( echo ${IP} | cut -d. -f1 )
    oct2=$( echo ${IP} | cut -d. -f2 )
    oct3=$( echo ${IP} | cut -d. -f3 )
    oct4=$( echo ${IP} | cut -d. -f4 )

    while read blacklistline ; do

      blacklist=`echo "${blacklistline}" | awk '{print $1}'`
      askdns=
      [ "`echo "${blacklistline}" | awk '{print $2}'`" = '1' ] && askdns=${DNServer}
      if host -tA -W$Time "${oct4}.${oct3}.${oct2}.${oct1}.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then
        TXT="`host -t txt -W$Time "${oct4}.${oct3}.${oct2}.${oct1}.${blacklist}." ${askdns} | grep -o 'descriptive text .*' | cut -b19- | sed 's/\"$//'`"
        [ -z "${Domain}" ] || echo -n "${Domain}: " >>$ftmp1
        if [ -z "${TXT}" ] ; then
          echo "${IP} is listed @ ${blacklist}" >>$ftmp1
        else
          echo "${TXT}" >>$ftmp1
        fi
      fi
    done <$ftmp3

  done
  if [ -s "$ftmp1" ] ; then
    if [ ${ZABBIX_SENDER} -eq 0 ] ; then
      cat $ftmp1 | tee -a ${reportfile}
    else
      cat $ftmp1 >>${reportfile}
      while read dnsbline ; do
        zsend chk "${dnsbline}"
      done <$ftmp1
    fi
    someonelisted=1
  fi
done <$ftmp2

if [ $someonelisted -eq 0 ] ; then
 [ ${ZABBIX_SENDER} -eq 0 ] && echo '.'
 [ ${ZABBIX_SENDER} -eq 0 ] || zsend chk "."
fi
_exit 0

unSpawn · 12-17-2010, 07:17 PM

Three questions: why would you want to RBL-check things afterwards? Why does the list you use offer match dnsbl.info closely (why not use their resources then?), and if you can not personally vouch for each and every RBL on that list you promote, why would you use it instead of a site that offers multi-RBL checks or use something like 'surblhost'? Just being curious.

frater · 12-18-2010, 02:09 AM

Quote:

Originally Posted by unSpawn

Three questions: why would you want to RBL-check things afterwards? Why does the list you use offer match dnsbl.info closely (why not use their resources then?), and if you can not personally vouch for each and every RBL on that list you promote, why would you use it instead of a site that offers multi-RBL checks or use something like 'surblhost'? Just being curious.

I don't want to rely on foreign sites besides the DNSBL's themselves. The list I'm using was offered in the thread of Zabbix where I started posting it. I've always been using the list that was offered by ASSP. As it turns out it's not a very good list...

But my focus at the moment is really not on that list. I think some of the things I'm doing in the script are quite original and effective and wanted to share it with all of you. I maybe should have waited a day more as I'm still working on it.

It should work on IP's on A-records and even on A-records resolving to more than 1 IP.

I appreciate your feedback!
BTW... I don't think I understood all your questions.. My brain is still in programming-mode

frater · 12-18-2010, 02:18 AM

Quote:

Originally Posted by unSpawn

Three questions: why would you want to RBL-check things afterwards?

I work for an ISP that allows their clients to run mailserver and this way we have an extra check on the reputation of these IP's.

The script is run every hour and it is using a list that's already there (it's generated by IPaudit the night before). If someone appears on this list we can investigate why he's on that list.
We have other ways as well to check for spamming clients.

grail · 12-18-2010, 06:10 AM

Well I can't say I understand all of the functions contained within but here does appear to be a deal of redundancy and repetition going on in the code.
One example would be at the end:

Code:

for IP in ${IPline} ; do
    while read blacklistline ; do
      oct1=$( echo ${IP} | cut -d. -f1 )
      oct2=$( echo ${IP} | cut -d. -f2 )
      oct3=$( echo ${IP} | cut -d. -f3 )
      oct4=$( echo ${IP} | cut -d. -f4 )

Here you have a for loop that populates IP and yet every time your while loop runs it cuts up the same IP again and again??

unSpawn · 12-18-2010, 07:28 AM

Quote:

Originally Posted by frater

I don't want to rely on foreign sites besides the DNSBL's themselves.

No, I mean that some sites can checks other RBLs which makes sense if a few of them are down or defunct.

Quote:

Originally Posted by frater

The list I'm using was offered in the thread of Zabbix where I started posting it. I've always been using the list that was offered by ASSP. As it turns out it's not a very good list... But my focus at the moment is really not on that list.

I'm only trying to make you aware that what you post will be read by tens or hundreds over time. Not all realize not every RBL has the same scope or quality and will just take things for granted.

Quote:

Originally Posted by frater

I think some of the things I'm doing in the script are quite original and effective and wanted to share it with all of you.

Sharing sure is appreciated!

frater · 12-18-2010, 08:41 AM

Quote:

Originally Posted by grail

Well I can't say I understand all of the functions contained within but here does appear to be a deal of redundancy and repetition going on in the code.
One example would be at the end:

Code:

for IP in ${IPline} ; do
    while read blacklistline ; do
      oct1=$( echo ${IP} | cut -d. -f1 )
      oct2=$( echo ${IP} | cut -d. -f2 )
      oct3=$( echo ${IP} | cut -d. -f3 )
      oct4=$( echo ${IP} | cut -d. -f4 )

Here you have a for loop that populates IP and yet every time your while loop runs it cuts up the same IP again and again??

You are totally right, but it's how the code evolved (or degenerated

)... That loop wasn't there before and I placed it a bit too early when I added it....
Thanks.... corrected it (just thought it was finished).

I do not agree with the redundancy of code although you were able to give that example. I even use it recursively when it needs to build a new DNSBL-list.

Big parts of the code are not executed. The program can be called with a lot of different parameters. The checking of the DNSBL-list will only be done once a day if it is found to be correct. Because it is intended to run on a lot of IP's it's important to test them with only valid DNSBL's or it will slow down alot.

BTW Do you know a good one-line for those 4-lines? I copied it from someone else's code, because I was too busy solving other problems (timeout problems with zabbix).

grail · 12-18-2010, 09:56 AM

Quote:

I even use it recursively when it needs to build a new DNSBL-list.

Yeah I wasn't quite sure how this worked?? I used your URL to pull down a list of 92 addresses.
What kind of tidy up would need to be done on that?

Quote:

BTW Do you know a good one-line for those 4-lines?

How about:

Code:

 set ${IP//./ }

Now each piece will be the corresponding - $1, $2, $3, $4

Other things I noticed:

1. Must this script be run by root?

Code:

export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

I would say yes based on adjusting PATH like so, and yet you have no test to confirm this?

2. Something personal, but I found it hard to keep track of parameters due to a lack of convention: UPPERCASE, lowercase, Capitalised

3. Again personal, but you can make arithmetic comparisons much clearer (IMO) by using the correct testing structure:

Code:

if [ ${LISTIPS} -eq 0 ]
# becomes
if (( LISTIPS == 0 ))

4. Why not use a single test when testing files exist:

Code:

if [ -e ${dnsbl} ] && [ ! -d ${dnsbl} ]
# should be
if [ -f $dnsbl ]

5. The following appears to not have any use as it changes no data either in a variable / file

Code:

grep -vf ${dnsbl} $ftmp3 | sed 's/\.$//' | sed "s/.*/host -tA -W$Time 2.0.0.127.&. ; host -tA -W$Time 1.0.0.127.&./"

6. Here you could use simple bash instead of goign out to external programs:

Code:

elif echo "${DNSBL}" | grep -q '/' ; then
# becomes
elif [[ $DNSBL =~ / ]]; then

7. Why the need for several programs when one can do all that you need?

Code:

cat "${DNSBL}" | awk '{print $1}' | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2

Firstly the cat is superfluous as awk / sed / grep can all use a filename as input. I would probably look at either sed or awk as either would easily handle this line on their own

8. Bad use of variable names the same as function names. Also if you are using a late enough bash (4+ I think) you can use parameter substitution:

Code:

dirname=`dirname ${DNSBL}`
basename=`basename ${DNSBL}`
TouchedToday="`find "${dirname}" -maxdepth 1 -name "${basename}"  -mtime 0`"
# becomes
TouchedToday="$(find "${DNSBL%/*}" -maxdepth 1 -name "${DNSBL##*/}"  -mtime 0)"

I also changed `` for $() as it is a lot clearer

9. This one just has me curious, what errors is cat likely to display?

Code:

lines1=`cat "${DNSBL}" 2>/dev/null | wc -l`

10. I presume you are aware that the following is not an accurate grep for only valid ip's:

Code:

grep -o 'remote_port=25&ip=[0-9.]*'

As it just as easily will match - 125489.2356898.3

11. This one is a bust too:

Code:

echo "${IP}" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*$'

As it will match - .123.23.45

frater · 12-18-2010, 02:37 PM

Quote:

Originally Posted by grail

Yeah I wasn't quite sure how this worked?? I used your URL to pull down a list of 92 addresses.
What kind of tidy up would need to be done on that?

I'm doing a DNS-request of 2.0.0.127.<blacklist> which should return a 127.0.0.0/8 address (I now do 127.0.0.0/24 but I need to do some investigating on that.
If that one succeeds it will do 1.0.0.127.<blacklist> which should fail.
Some blacklist providers that are about to shut down their service set both to 127.0.0.2.

Quote:

Originally Posted by grail

How about:

Code:

 set ${IP//./ }

Now each piece will be the corresponding - $1, $2, $3, $4

I think I'm missing something here. Can you elaborate on that? The octets do need to be in reverse order

Quote:

Originally Posted by grail

Other things I noticed:

1. Must this script be run by root?

Code:

export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

I would say yes based on adjusting PATH like so, and yet you have no test to confirm this?

Yes as root or as zabbix using sudo
Timing turned out to be an issue when using it with the agent. So it's now an hourly cronjob which actively sends the info to the Zabbix-server (with zabbix-sender)

Quote:

Originally Posted by grail

2. Something personal, but I found it hard to keep track of parameters due to a lack of convention: UPPERCASE, lowercase, Capitalised

Yes, you're right. I started programming in a language where you don't have to define the variables beforehand.

Quote:

Originally Posted by grail

3. Again personal, but you can make arithmetic comparisons much clearer (IMO) by using the correct testing structure:

Code:

if [ ${LISTIPS} -eq 0 ]
# becomes
if (( LISTIPS == 0 ))

4. Why not use a single test when testing files exist:

Code:

if [ -e ${dnsbl} ] && [ ! -d ${dnsbl} ]
# should be
if [ -f $dnsbl ]

It's done on purpose, I also want to support symbolic links (if I want to use the same file for DNSBL as my ASSP)

Quote:

Originally Posted by grail

5. The following appears to not have any use as it changes no data either in a variable / file

Code:

grep -vf ${dnsbl} $ftmp3 | sed 's/\.$//' | sed "s/.*/host -tA -W$Time 2.0.0.127.&. ; host -tA -W$Time 1.0.0.127.&./"

It was a cosmetic touch. It gives you all the failed DNSBL's and with your terminal-client (putty?) you then only need to mark it and press right-click to do the test manually.

Quote:

Originally Posted by grail

6. Here you could use simple bash instead of goign out to external programs:

Code:

elif echo "${DNSBL}" | grep -q '/' ; then
# becomes
elif [[ $DNSBL =~ / ]]; then

I will probably use that syntax in the future.

Quote:

Originally Posted by grail

7. Why the need for several programs when one can do all that you need?

Code:

cat "${DNSBL}" | awk '{print $1}' | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2

Firstly the cat is superfluous as awk / sed / grep can all use a filename as input. I would probably look at either sed or awk as either would easily handle this line on their own

I still need to study on awk to use it as it was intended. I always use simple awk commands that I pipe. It's somehow simpler for me to use it the way I do it.

Quote:

Originally Posted by grail

8. Bad use of variable names the same as function names. Also if you are using a late enough bash (4+ I think) you can use parameter substitution:

Code:

dirname=`dirname ${DNSBL}`
basename=`basename ${DNSBL}`
TouchedToday="`find "${dirname}" -maxdepth 1 -name "${basename}"  -mtime 0`"
# becomes
TouchedToday="$(find "${DNSBL%/*}" -maxdepth 1 -name "${DNSBL##*/}"  -mtime 0)"

I also changed `` for $() as it is a lot clearer

I agree with almost anything you say, but I really prefer backticks.

Quote:

Originally Posted by grail

9. This one just has me curious, what errors is cat likely to display?

Code:

lines1=`cat "${DNSBL}" 2>/dev/null | wc -l`

At the time of writing I probably thought the file could be missing.

Quote:

Originally Posted by grail

10. I presume you are aware that the following is not an accurate grep for only valid ip's:

Code:

grep -o 'remote_port=25&ip=[0-9.]*'

As it just as easily will match - 125489.2356898.3

It wasn't meant for sanitizing. It's just a quick way to extract the IP of outgoing SMTP-server out of the IPaudit html-file

Quote:

Originally Posted by grail

11. This one is a bust too:

Code:

echo "${IP}" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*$'

As it will match - .123.23.45

I use that syntax often with 'grep -o' where it works great. You know what I intend. Could you please improve on that syntax? I believe perl uses a '+' sign.....

Mmmm... Answered my own question here:

Code:

# echo 1.123.23.45 | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
1.123.23.45
# echo .123.23.45 | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
#

Maybe you can still improve on it?

frater · 12-18-2010, 02:59 PM

I'm posting the changes in a seperate post.
I have a feeling some more patches may come.
It seems to do everything it should do.

cat /usr/local/sbin/check_rbl

Code:

#!/bin/bash
#####################################################
# check_rbl
#####################################################
# 17-12-2010 by JP van Melis
#
# Use at your own risk!
#####################################################
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

# If you don't want to use your own DNS-server
# Use Google's, Blank it if you want to use the default always.
DNServer=8.8.8.8
# URL=http://blog.penumbra.be/files/zabbix/rbl_list.txt
URL=http://wd.mirmana.com/dnsbl.txt
dnsbl=/etc/dnsbl.txt
IPAUDITDIR=/home/ipaudit/reports/daily/server2/html

DNSBL=
TESTDNSBL=0
LISTIPS=0
ZABBIX_SENDER=0
ZABBIX_CONF=/etc/zabbix/zabbix_agentd.conf

TouchedToday=
Time=2
DEBUG=0

NAME="$0"
[ -z "`which readlink`" ] || NAME="`readlink -f "${NAME}"`"
BASENAME=`basename "${NAME}"`
reportfile=/var/log/${BASENAME}.report

while getopts zltr:d: name
do
  case $name in
    z)   ZABBIX_SENDER=1;;
    l)   LISTIPS=1;;
    t)   TESTDNSBL=1;;
    r)   reportfile="$OPTARG";;
    d)   DNSBL="$OPTARG";;
    ?)   printf "Usage: %s: [-d] [-l] [-t] [-r <reportfile>] [(<IP>|<domain>|<file>)]\n" $0
    exit 2;;
  esac
done
shift $(($OPTIND - 1))

_exit ()
{
  rm -f $ftmp1 2>/dev/null
  rm -f $ftmp2 2>/dev/null
  rm -f $ftmp3 2>/dev/null
  exit $1
}

if [ ${ZABBIX_SENDER} -eq 1 ] ; then
  [ -e ${ZABBIX_CONF} ] || ZABBIX_CONF=/opt${ZABCONF}
  if [ ! -e ${ZABBIX_CONF} ] ; then
    echo "Unable to find ${ZABBIX_CONF}, can't continue" >&2
    exit 1
  fi
  # zabHOSTNAME=`grep -i ^Hostname ${ZABBIX_CONF} | awk -F= '{print $2}' | awk '{print $1}'`
fi

ftmp1=`mktemp`
ftmp2=`mktemp`
ftmp3=`mktemp`

if [ ${LISTIPS} -eq 0 ] ; then

  # No DNSBL given... use standard file or create one by recursively calling myself
  [ -z "${DNSBL}" ] && [ -e ${dnsbl} ] && [ ! -d ${dnsbl} ] && DNSBL="${dnsbl}"
  if [ -z "${DNSBL}" ] ; then
    echo -e "No ${dnsbl} found...\nI will make an attempt to create one by downloading ${URL}!"
    if wget -O $ftmp3 ${URL} 2>/dev/null ; then
      lines1=`cat $ftmp3 | wc -l`
      # make it look older, so it will get checked next run
      touch -d "last week" $ftmp3
      echo "Downloaded ${lines1} DNSBL's, I will now recursively call myself to clean the file up!"
      $0 -d $ftmp3 -t >${dnsbl}
      lines2=`cat ${dnsbl} | wc -l`
      if [ $lines2 -eq $lines1 ] ; then
        echo "All DNSBL's seem to be valid!"
      else
        echo -e "Only $lines2 survived the test!\n\nThese were found to be invalid:\n"
        while read failed ; do
          grep -q "${failed}" ${dnsbl} || echo "${failed}" | awk '{print $1}' | sed 's/\.$//' | sed "s/.*/host -tA -W$Time 2.0.0.127.&. ; host -tA -W$Time 1.0.0.127.&./"
        done <$ftmp3
      fi
      echo -e "\nThe default DNSBL list (${dnsbl}) is created, you should not see this anymore"
      _exit 1
    else
      echo "Download from ${URL} failed"
      _exit 1
    fi
  elif  [[ $DNSBL =~ / ]]; then
    # Create list from file
    if [ -e "${DNSBL}" ] && [ ! -d "${DNSBL}" ] ; then
      if head -n1 "${DNSBL}" | grep -q '^[-0-9A-Za-z_.]* [01]$' ; then
        cp -p "${DNSBL}" $ftmp2
        TouchedToday="$(find "${DNSBL%/*}" -maxdepth 1 -name "${DNSBL##*/}"  -mtime 0)"
      else
        awk '{print $1}' "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
      fi
    fi
  else
    # Create list from command-line parameter
    echo "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
  fi

  # if file is unchecked or its first line doesn't end with ' 0' oe ' 1'
  if [ -z "${TouchedToday}" ] ; then
    # Check if DNSBL-server is working
    while read blacklistline ; do
      # remove a trailing dot, so we can add one without making a duplicate
      blacklist=`echo "${blacklistline}" | awk '{print $1}' | sed 's/\.$//'`
      if host -tA -W$Time 2.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.' ; then
        host -tA -W$Time 1.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 1" >>${ftmp3}
      elif [ ! -z "${DNServer}" ] ; then
        if host -tA -W$Time 2.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.' ; then
          host -tA -W$Time 1.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 0" >>${ftmp3}
        fi
      fi
    done<${ftmp2}
  else
    cp -p $ftmp2 $ftmp3
  fi

  if [ ! -s ${ftmp3} ] ; then
    echo "No valid DNSBL-servers found"
    _exit 1
  elif echo "${DNSBL}" | grep -q '/' ; then
    if [ -z "${TouchedToday}" ] ; then
      # If a file is used for the DNSBL's then check if all are valid..
      # If so, then 'touch it', so it doesn't need checking today.
      lines1=`cat "${DNSBL}" | wc -l`
      lines2=`cat "${ftmp3}" | wc -l`
      [ $lines1 -eq $lines2 ] && touch "${DNSBL}"
    fi
  fi

  if [ $TESTDNSBL -eq 1 ] ; then
    sort -u ${ftmp3}
    if [ ${ZABBIX_SENDER} -eq 1 ] ; then
      DNSBLS="`cat ${ftmp3}`"
      zabbix_sender -c $ZABBIX_CONF -k "dnsbl[dnsbl]" -o "${DNSBLS}" 2>&1 >/dev/null
    fi
    _exit 0
  fi

  [ -s ${ftmp3} ] || exit 1
fi

# delete surrounding spaces
IPSTOTEST="`echo "$*" | sed 's/^ *//;s/ *$//'`"

if [ -z "${IPSTOTEST}" ] ; then
  if [ -d "${IPAUDITDIR}" ] ; then
    # check IPaudit's daily list
    fname=`ls -1t ${IPAUDITDIR}/*.html 2>/dev/null | head -n1`
    [ -e "${fname}" ] && grep -o 'remote_port=25&ip=[0-9.]*' "${fname}" |  awk -F= '{print $3}' | sed 's/^0*//;s/\.0*/./g' >$ftmp1
  else
    wget -O - http://checkip.dyndns.org/ 2>/dev/null | grep -io 'IP Address: [0-9.]*' | awk '{print $3}' >$ftmp1
  fi
elif echo "${IPSTOTEST}" | grep -q '/' ; then
  [ -e "${IPSTOTEST}" ] && grep -o '[-0-9A-Za-z_.]*' "${IPSTOTEST}" | sed 's/\.$//' >$ftmp1
else
  echo "${IPSTOTEST}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp1
fi

# Convert host names to IP's if it's not done already.
echo -n '' >$ftmp2
while read IP ; do
  if echo "${IP}" | egrep -q '^([0-9]{1,3}\.){3}[0-9]{1,3}$' ; then
    echo "${IP}" >>$ftmp2
  else
    # extract all IP's and concatenate domain to it
    ResolvedIP="`host -tA "${IP}." | grep -o 'has address [0-9.]*' | awk '{printf "%s ", $3}'`"
    [ -z "${ResolvedIP}" ] || echo "${ResolvedIP}${IP}." >>$ftmp2
  fi
done <$ftmp1

if [ ! -s ${ftmp2} ] ; then
  echo "NO IP's found" 2>/dev/null
  _exit 1
fi

if [ ${LISTIPS} -eq 1 ] ; then
  cat ${ftmp2}

  if [ ${ZABBIX_SENDER} -eq 1 ] ; then
    IPS="`cat ${ftmp2}`"
    zabbix_sender -c $ZABBIX_CONF -k "dnsbl[ips]" -o "${IPS}" 2>&1 >/dev/null
  fi
  _exit 0
fi

echo -n '' >${reportfile}
while read IPline ; do

  echo -n '' >$ftmp1

  # extract Domain from IPline and cut that Domain out of IPline
  Domain="`echo "${IPline}" | sed -e 's/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*//g' | sed 's/ *//g'`"
  subject="`echo "${Domain}:" | awk '{ printf("%-25s",$0) }'`"
  [ -z "${Domain}" ] || IPline="`echo "${IPline}" | sed -e "s/${Domain}//"`"

  for IP in ${IPline} ; do

    set ${IP//./ }
    [ -z "${Domain}" ] && subject="`echo "${IP}:" | awk '{ printf("%-18s\n",$0) }'`"

    while read blacklistline ; do

      blacklist=`echo "${blacklistline}" | awk '{print $1}'`
      askdns=
      echo "${blacklistline}" | grep -q ' 1$' && askdns=${DNServer}

      if host -tA -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

        TXT="`host -t txt -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -o 'descriptive text .*' | cut -b19- | sed 's/\"$//'`"
        echo -n "${subject} ${TXT}" >>$ftmp1
        [ -z "${TXT}" ] && echo -n "listed @ ${blacklist}" >>$ftmp1
        echo '' >>$ftmp1

      fi
    done <$ftmp3
  done
  cat $ftmp1 | tee -a ${reportfile}
done <$ftmp2

if [ -s ${reportfile} ] ; then
  IPS="`cat ${reportfile}`"
  zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "${IPS}" 2>&1 >/dev/null
else
  echo '.'
  [ ${ZABBIX_SENDER} -eq 1 ] && zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "." 2>&1 >/dev/null
fi
_exit 0

grail · 12-18-2010, 11:59 PM

Quote:

Originally Posted by frater

Quote:
Originally Posted by grail
How about:
Code:
set ${IP//./ }
Now each piece will be the corresponding - $1, $2, $3, $4
I think I'm missing something here. Can you elaborate on that? The octets do need to be in reverse order

You can put these in any order you wish. All this has done is instead of going through all the cuts you can now reference each by parameter position.

Code:

set ${IP//./ }

# and then this
if host -tA -W$Time "${oct4}.${oct3}.${oct2}.${oct1}.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

# would now look like
if host -tA -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

As for improving the grep, not sure this is it but maybe worth a look:

Code:

echo 1.123.23.45 | egrep -o '([0-9]{1,3}\.){3}[0-9]{1,3}'

Of course egrep is an synonym for grep -E

frater · 12-19-2010, 02:37 AM

Quote:

Originally Posted by grail

You can put these in any order you wish. All this has done is instead of going through all the cuts you can now reference each by parameter position.

Code:

set ${IP//./ }

# and then this
if host -tA -W$Time "${oct4}.${oct3}.${oct2}.${oct1}.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

# would now look like
if host -tA -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

How nifty. I didn't know that use of set. Thanks, I will be using this some more. It does mean I will lose my parameters, but I'm not using them anymore after that. I'm glad I asked you this.

Quote:

Originally Posted by grail

As for improving the grep, not sure this is it but maybe worth a look:

Code:

echo 1.123.23.45 | egrep -o '([0-9]{1,3}\.){3}[0-9]{1,3}'

Of course egrep is an synonym for grep -E

I know that expression from ASSP. He's using it all the time, but that's in perl.
So it seems I can use it in bash as well. :-D
Back to the drawing board....

frater · 12-31-2010, 07:09 AM

A lot of thanks goes out to 'grail'
This is what I'm using now myself:

Code:

#!/bin/bash
#####################################################
# check_rbl
#####################################################
# 17-12-2010 by JP van Melis
#
# Use at your own risk!
#####################################################
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

# If you don't want to use your own DNS-server
# Use Google's, Blank it if you want to use the default always.
DNServer=8.8.8.8
# URL=http://blog.penumbra.be/files/zabbix/rbl_list.txt
URL=http://wd.mirmana.com/dnsbl.txt
dnsbl=/etc/dnsbl.txt
IPAUDITDIR=/home/ipaudit/reports/daily/server2/html

DNSBL=
TESTDNSBL=0
LISTIPS=0
ZABBIX_SENDER=0
ZABBIX_CONF=/etc/zabbix/zabbix_agentd.conf

TouchedToday=
Time=2
DEBUG=0

NAME="$0"
[ -z "`which readlink`" ] || NAME="`readlink -f "${NAME}"`"
BASENAME=`basename "${NAME}"`
reportfile=/var/log/${BASENAME}.report

while getopts zltr:d: name
do
  case $name in
    z)   ZABBIX_SENDER=1;;
    l)   LISTIPS=1;;
    t)   TESTDNSBL=1;;
    r)   reportfile="$OPTARG";;
    d)   DNSBL="$OPTARG";;
    ?)   printf "Usage: %s: [-d] [-l] [-t] [-r <reportfile>] [(<IP>|<domain>|<file>)]\n" $0
    exit 2;;
  esac
done
shift $(($OPTIND - 1))

_exit ()
{
  rm -f $ftmp1 2>/dev/null
  rm -f $ftmp2 2>/dev/null
  rm -f $ftmp3 2>/dev/null
  rm -f $ftmp4 2>/dev/null
  exit $1
}

if [ ${ZABBIX_SENDER} -eq 1 ] ; then
  [ -e ${ZABBIX_CONF} ] || ZABBIX_CONF=/opt${ZABCONF}
  if [ ! -e ${ZABBIX_CONF} ] ; then
    echo "Unable to find ${ZABBIX_CONF}, can't continue" >&2
    exit 1
  fi
  # zabHOSTNAME=`grep -i ^Hostname ${ZABBIX_CONF} | awk -F= '{print $2}' | awk '{print $1}'`
fi

ftmp1=`mktemp`
ftmp2=`mktemp`
ftmp3=`mktemp`
ftmp4=`mktemp`

if [ ${LISTIPS} -eq 0 ] ; then

  # No DNSBL given... use standard file or create one by recursively calling myself
  [ -z "${DNSBL}" ] && [ -e ${dnsbl} ] && [ ! -d ${dnsbl} ] && DNSBL="${dnsbl}"
  if [ -z "${DNSBL}" ] ; then
    echo -e "No ${dnsbl} found...\nI will make an attempt to create one by downloading ${URL}!"
    if wget -O $ftmp3 ${URL} 2>/dev/null ; then
      lines1=`cat $ftmp3 | wc -l`
      # make it look older, so it will get checked next run
      touch -d "last week" $ftmp3
      echo "Downloaded ${lines1} DNSBL's, I will now recursively call myself to clean the file up!"
      $0 -d $ftmp3 -t >${dnsbl}
      lines2=`cat ${dnsbl} | wc -l`
      if [ $lines2 -eq $lines1 ] ; then
        echo "All DNSBL's seem to be valid!"
      else
        echo -e "Only $lines2 survived the test!\n\nThese were found to be invalid:\n"
        while read failed ; do
          grep -q "${failed}" ${dnsbl} || echo "${failed}" | awk '{print $1}' | sed 's/\.$//' | sed "s/.*/host -tA -W$Time 2.0.0.127.&. ; host -tA -W$Time 1.0.0.127.&./"
        done <$ftmp3
      fi
      echo -e "\nThe default DNSBL list (${dnsbl}) is created, you should not see this anymore"
      _exit 1
    else
      echo "Download from ${URL} failed"
      _exit 1
    fi
  elif  [[ $DNSBL =~ / ]]; then
    # Create list from file
    if [ -e "${DNSBL}" ] && [ ! -d "${DNSBL}" ] ; then
      if head -n1 "${DNSBL}" | grep -q '^[-0-9A-Za-z_.]* [01]$' ; then
        cp -p "${DNSBL}" $ftmp2
        TouchedToday="$(find "${DNSBL%/*}" -maxdepth 1 -name "${DNSBL##*/}"  -mtime 0)"
      else
        awk '{print $1}' "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
      fi
    fi
  else
    # Create list from command-line parameter
    echo "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
  fi

  # if file is unchecked or its first line doesn't end with ' 0' oe ' 1'
  if [ -z "${TouchedToday}" ] ; then
    # Check if DNSBL-server is working
    while read blacklistline ; do
      # remove a trailing dot, so we can add one without making a duplicate
      blacklist=`echo "${blacklistline}" | awk '{print $1}' | sed 's/\.$//'`
      if host -tA -W$Time 2.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.' ; then
        host -tA -W$Time 1.0.0.127.${blacklist}. ${DNServer} | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 1" >>${ftmp3}
      elif [ ! -z "${DNServer}" ] ; then
        if host -tA -W$Time 2.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.' ; then
          host -tA -W$Time 1.0.0.127.${blacklist}. | grep -q 'has address 127\.0\.0\.2' || echo "${blacklist} 0" >>${ftmp3}
        fi
      fi
    done<${ftmp2}
  else
    cp -p $ftmp2 $ftmp3
  fi

  if [ ! -s ${ftmp3} ] ; then
    echo "No valid DNSBL-servers found"
    _exit 1
  elif echo "${DNSBL}" | grep -q '/' ; then
    if [ -z "${TouchedToday}" ] ; then
      # If a file is used for the DNSBL's then check if all are valid..
      # If so, then 'touch it', so it doesn't need checking today.
      lines1=`cat "${DNSBL}" | wc -l`
      lines2=`cat "${ftmp3}" | wc -l`
      [ $lines1 -eq $lines2 ] && touch "${DNSBL}"
    fi
  fi

  if [ $TESTDNSBL -eq 1 ] ; then
    sort -u ${ftmp3}
    if [ ${ZABBIX_SENDER} -eq 1 ] ; then
      DNSBLS="`cat ${ftmp3}`"
      zabbix_sender -c $ZABBIX_CONF -k "dnsbl[dnsbl]" -o "${DNSBLS}" 2>&1 >/dev/null
    fi
    _exit 0
  fi

  [ -s ${ftmp3} ] || exit 1
fi

# delete surrounding spaces
IPSTOTEST="`echo "$*" | sed 's/^ *//;s/ *$//'`"

if [ -z "${IPSTOTEST}" ] ; then
  if [ -d "${IPAUDITDIR}" ] ; then
    # check IPaudit's daily list
    fname=`ls -1t ${IPAUDITDIR}/*.html 2>/dev/null | head -n1`
    [ -e "${fname}" ] && grep -o 'remote_port=25&ip=[0-9.]*' "${fname}" |  awk -F= '{print $3}' | sed 's/^0*//;s/\.0*/./g' >$ftmp1
  else
    wget -O - http://checkip.dyndns.org/ 2>/dev/null | grep -io 'IP Address: [0-9.]*' | awk '{print $3}' >$ftmp1
  fi
elif echo "${IPSTOTEST}" | grep -q '/' ; then
  [ -e "${IPSTOTEST}" ] && grep -o '[-0-9A-Za-z_.]*' "${IPSTOTEST}" | sed 's/\.$//' >$ftmp1
else
  echo "${IPSTOTEST}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp1
fi

# Convert host names to IP's if it's not done already.
echo -n '' >$ftmp2
while read IP ; do
  if echo "${IP}" | egrep -q '^([0-9]{1,3}\.){3}[0-9]{1,3}$' ; then
    echo "${IP}" >>$ftmp2
  else
    # extract all IP's and concatenate domain to it
    ResolvedIP="`host -tA "${IP}." | grep -o 'has address [0-9.]*' | awk '{printf "%s ", $3}'`"
    [ -z "${ResolvedIP}" ] || echo "${ResolvedIP}${IP}." >>$ftmp2
  fi
done <$ftmp1

if [ ! -s ${ftmp2} ] ; then
  echo "NO IP's found" 2>/dev/null
  _exit 1
fi

sort ${ftmp2} -o ${ftmp4}

if [ ${LISTIPS} -eq 1 ] ; then
  cat ${ftmp4}

  if [ ${ZABBIX_SENDER} -eq 1 ] ; then
    IPS="`cat ${ftmp4}`"
    zabbix_sender -c $ZABBIX_CONF -k "dnsbl[ips]" -o "${IPS}" 2>&1 >/dev/null
  fi
  _exit 0
fi

echo -n '' >${reportfile}
while read IPline ; do

  echo -n '' >$ftmp1

  # extract Domain from IPline and cut that Domain out of IPline
  Domain="`echo "${IPline}" | sed -e 's/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*//g' | sed 's/ *//g'`"
  subject="`echo "${Domain}:" | awk '{ printf("%-25s",$0) }'`"
  [ -z "${Domain}" ] || IPline="`echo "${IPline}" | sed -e "s/${Domain}//"`"

  for IP in ${IPline} ; do

    set ${IP//./ }
    [ -z "${Domain}" ] && subject="`echo "${IP}:" | awk '{ printf("%-18s\n",$0) }'`"

    while read blacklistline ; do

      blacklist=`echo "${blacklistline}" | awk '{print $1}'`
      askdns=
      echo "${blacklistline}" | grep -q ' 1$' && askdns=${DNServer}

      if host -tA -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -q 'has address 127\.0\.0\.' ; then

        TXT="`host -t txt -W$Time "$4.$3.$2.$1.${blacklist}." ${askdns} | grep -o 'descriptive text .*' | cut -b19- | sed 's/\"$//'`"
        echo -n "${subject} ${TXT}" >>$ftmp1
        [ -z "${TXT}" ] && echo -n "listed @ ${blacklist}" >>$ftmp1
        echo '' >>$ftmp1

      fi
    done <$ftmp3
  done
  cat $ftmp1 | tee -a ${reportfile}
done <$ftmp4

if [ -s ${reportfile} ] ; then
  IPS="`cat ${reportfile}`"
  zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "${IPS}" 2>&1 >/dev/null
else
  echo '.'
  [ ${ZABBIX_SENDER} -eq 1 ] && zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "." 2>&1 >/dev/null
fi
_exit 0

grail · 12-31-2010, 08:14 AM

I have one suggestion and one personal niggle

Suggestion: No need to go to an external app for such a trivial substitution:

Code:

blacklist=`echo "${blacklistline}" | awk '{print $1}'`

# becomes

blacklist=${blacklistline%% *}

The niggle is about your _exit function positioning. I find it breaks the flow to be into the main part of the script and then suddenly define random functions.
As we know the issue with bash is it is a top down language (ie function must be declared prior to first use). So to this end I prefer to place all my
functions at the start followed by some delimiter to notify that the functions section has completed.

Again this one is just personal ... looks like you are having fun though

frater · 01-01-2011, 04:52 AM

Best wishes for the next year, my teacher

I decided to clean the file and make it more human-readable.
I still need to study on those Bash-statements like

Code:

REPORTFILE=/var/log/${NAME##*/}.report

They are shorter, but not really more straightforward.

I think it looks quite nice now, but I'm sure you can find something that could be changed. ;-)
I even thought about making this loop a sed one-liner...

Code:

while read BLDOMAINLINE ; do    # Show all failed DNSBL's in a format you can execute on the command-line
  BLDOMAIN=`echo "${BLDOMAINLINE%% *}" | sed 's/\.$//'`
  if ! grep -q "${BLDOMAIN}" ${DNSBLFILE} ; then
    echo "${BLDOMAIN}" | sed "s/.*/host -tA -W$TIMEOUT 2.0.0.127.&. ; host -tA -W$TIMEOUT 1.0.0.127.&./"
  fi
done <$ftmp3

BTW...
A google search of deleting empty lines in a file gave me all kind of complicated solutions.
This one is best for doing this, isn't it?
I also want to remove files holding only spaces

Code:

sed -i '/^ *$/d' $ftmp3

Code:

#!/bin/bash
#####################################################
# check_rbl
#####################################################
# 17-12-2010 by JP van Melis
#
# Use at your own risk!
#####################################################
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

# Functions
#-----------------------
_exit ()
{
  rm -f $ftmp1 2>/dev/null
  rm -f $ftmp2 2>/dev/null
  rm -f $ftmp3 2>/dev/null
  rm -f $ftmp4 2>/dev/null
  exit $1
}

#-----------------------

# Constants
#-----------------------
URL=http://wd.mirmana.com/dnsbl.txt
# URL=http://blog.penumbra.be/files/zabbix/rbl_list.txt
DNSBLFILE=/etc/dnsbl.txt
IPAUDITDIR=/home/ipaudit/reports/daily/server2/html

DNS_SERVER=8.8.8.8                              # First DNS-server to try
ZABBIX_CONF=/etc/zabbix/zabbix_agentd.conf      # Config-file of the zabbix agent
TIMEOUT=2                                       # Timeout for each DNS-request
DEBUG=0                                         # Turn debugging on to troubleshoot
NAME="$0"

# Flags
#-----------------------
DNSBL=
TESTDNSBL=0
LISTIPS=0
ZABBIX_SENDER=0
TOUCHEDTODAY=
#-----------------------

# Create the name for the report file
#-----------------------
[ -z "`which readlink`" ] || NAME="`readlink -f "${NAME}"`"     # Maybe you don't want this if you have multiple checks
REPORTFILE=/var/log/${NAME##*/}.report
#-----------------------

# Handle the options
#-----------------------
while getopts zltr:d: name
do
  case $name in
    z)   ZABBIX_SENDER=1;;
    l)   LISTIPS=1;;
    t)   TESTDNSBL=1;;
    r)   REPORTFILE="$OPTARG";;
    d)   DNSBL="$OPTARG";;
    ?)   printf "Usage: %s: [-z] [-d] [-l] [-t] [-r <reportfile>] [(<IP>|<domain>|<file>)]\n" $0
    exit 2;;
  esac
done
shift $(($OPTIND - 1))
#-----------------------

# If Zabbix is used (-z), we NEED its config!
if [ ${ZABBIX_SENDER} -eq 1 ] ; then
  [ -e ${ZABBIX_CONF} ] || ZABBIX_CONF=/opt${ZABCONF}
  if [ ! -e ${ZABBIX_CONF} ] ; then
    echo "Unable to find ${ZABBIX_CONF}, can't continue" >&2
    exit 1
  fi
fi

# I guess the show is on!
ftmp1=`mktemp`
ftmp2=`mktemp`
ftmp3=`mktemp`
ftmp4=`mktemp`

if [ ${LISTIPS} -eq 0 ] ; then

  # No DNSBL given... use standard file or create one by recursively calling myself
  [ -z "${DNSBL}" ] && [ -e ${DNSBLFILE} ] && [ ! -d ${DNSBLFILE} ] && DNSBL="${DNSBLFILE}"
  if [ -z "${DNSBL}" ] ; then
    echo -e "No ${DNSBLFILE} found...\nI will make an attempt to create one by downloading ${URL}!"
    if wget -O $ftmp3 ${URL} 2>/dev/null ; then
      sed -i '/^ *$/d' $ftmp3           # Remove blank lines or lines only holding spaces
      L1=`cat $ftmp3 | wc -l`           # Count the lines
      touch -d "last week" $ftmp3       # make it look older, so it will get checked next run
      echo "Downloaded ${lines1} DNSBL's, I will now recursively call myself to clean the file up!"
      $0 -d $ftmp3 -t >${DNSBLFILE}     # Use the option "-d" to generate a sanitized and tested blacklist
      L2=`cat ${DNSBLFILE} | wc -l`     # Count the lines of new file
      if [ $L2 -eq $L1 ] ; then
        echo "All DNSBL's seem to be valid!"
      else
        echo -e "Only $L2 survived the test!\n\nThese were found to be invalid:\n"
        while read BLDOMAINLINE ; do    # Show all failed DNSBL's in a format you can execute on the command-line
          BLDOMAIN=`echo "${BLDOMAINLINE%% *}" | sed 's/\.$//'`
          if ! grep -q "${BLDOMAIN}" ${DNSBLFILE} ; then
            echo "${BLDOMAIN}" | sed "s/.*/host -tA -W$TIMEOUT 2.0.0.127.&. ; host -tA -W$TIMEOUT 1.0.0.127.&./"
          fi
        done <$ftmp3
      fi
      echo -e "\nThe default DNSBL list (${dnsbl}) is created, you should not see this anymore"
      _exit 1
    else
      echo "Download from ${URL} failed"
      _exit 1
    fi
  elif  [[ $DNSBL =~ / ]]; then
    # Create list from file
    if [ -e "${DNSBL}" ] && [ ! -d "${DNSBL}" ] ; then
      if head -n1 "${DNSBL}" | grep -q '^[-0-9A-Za-z_.]* [01]$' ; then
        cp -p "${DNSBL}" $ftmp2
        TOUCHEDTODAY="$(find "${DNSBL%/*}" -maxdepth 1 -name "${DNSBL##*/}"  -mtime 0)"
      else
        awk '{print $1}' "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
      fi
    fi
  else
    # Create list from command-line parameter
    echo "${DNSBL}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp2
  fi

  # if file is unchecked or its first line doesn't end with ' 0' oe ' 1'
  if [ -z "${TOUCHEDTODAY}" ] ; then
    while read BLDOMAINLINE ; do                                # Check if DNSBL-server is working
      BLDOMAIN=`echo "${BLDOMAINLINE%% *}" | sed 's/\.$//'`     # remove any trailing dot
      if host -tA -W$TIMEOUT 2.0.0.127.${BLDOMAIN}. ${DNS_SERVER} | grep -q 'has address 127\.0\.0\.' ; then
        host -tA -W$TIMEOUT 1.0.0.127.${BLDOMAIN}. ${DNS_SERVER} | grep -q 'has address 127\.0\.0\.2' || echo "${BLDOMAIN} 1" >>${ftmp3}
      elif [ ! -z "${DNS_SERVER}" ] ; then
        if host -tA -W$TIMEOUT 2.0.0.127.${BLDOMAIN}. | grep -q 'has address 127\.0\.0\.' ; then
          host -tA -W$TIMEOUT 1.0.0.127.${BLDOMAIN}. | grep -q 'has address 127\.0\.0\.2' || echo "${BLDOMAIN} 0" >>${ftmp3}
        fi
      fi
    done<${ftmp2}
  else
    cp -p $ftmp2 $ftmp3
  fi

  if [ ! -s ${ftmp3} ] ; then
    echo "No valid DNSBL-servers found"
    _exit 1
  elif echo "${DNSBL}" | grep -q '/' ; then
    if [ -z "${TOUCHEDTODAY}" ] ; then
      # If a file is used for the DNSBL's then check if all are valid..
      # If so, then 'touch it', so it doesn't need checking today.
      lines1=`cat "${DNSBL}" | wc -l`
      lines2=`cat "${ftmp3}" | wc -l`
      [ $lines1 -eq $lines2 ] && touch "${DNSBL}"
    fi
  fi

  if [ $TESTDNSBL -eq 1 ] ; then
    sort -u ${ftmp3}
    if [ ${ZABBIX_SENDER} -eq 1 ] ; then
      DNSBLS="`cat ${ftmp3}`"
      zabbix_sender -c $ZABBIX_CONF -k "dnsbl[dnsbl]" -o "${DNSBLS}" 2>&1 >/dev/null
    fi
    _exit 0
  fi

  [ -s ${ftmp3} ] || exit 1
fi

# delete surrounding spaces
IPSTOTEST="`echo "$*" | sed 's/^ *//;s/ *$//'`"

if [ -z "${IPSTOTEST}" ] ; then
  if [ -d "${IPAUDITDIR}" ] ; then
    # check IPaudit's daily list
    fname=`ls -1t ${IPAUDITDIR}/*.html 2>/dev/null | head -n1`
    [ -e "${fname}" ] && grep -o 'remote_port=25&ip=[0-9.]*' "${fname}" |  awk -F= '{print $3}' | sed 's/^0*//;s/\.0*/./g' >$ftmp1
  else
    wget -O - http://checkip.dyndns.org/ 2>/dev/null | grep -io 'IP Address: [0-9.]*' | awk '{print $3}' >$ftmp1
  fi
elif echo "${IPSTOTEST}" | grep -q '/' ; then
  [ -e "${IPSTOTEST}" ] && grep -o '[-0-9A-Za-z_.]*' "${IPSTOTEST}" | sed 's/\.$//' >$ftmp1
else
  echo "${IPSTOTEST}" | grep -o '[-0-9A-Za-z_.]*' | sed 's/\.$//' >$ftmp1
fi

# Convert host names to IP's if it's not done already.
echo -n '' >$ftmp2
while read IP_OR_DOMAIN ; do
  if echo "${IP_OR_DOMAIN}" | egrep -q '^([0-9]{1,3}\.){3}[0-9]{1,3}$' ; then
    echo "${IP_OR_DOMAIN}" >>$ftmp2
  else
    # resolve domain to IP(s)
    IPS="`host -tA "${IP_OR_DOMAIN}." | grep -o 'has address [0-9.]*' | awk '{printf "%s ", $3}'`"
    [ -z "${IPS}" ] || echo "${IPS}${IP_OR_DOMAIN}." >>$ftmp2
  fi
done <$ftmp1

if [ ! -s ${ftmp2} ] ; then
  echo "NO IP's found" 2>/dev/null
  _exit 1
fi

sort ${ftmp2} -o ${ftmp4}

if [ ${LISTIPS} -eq 1 ] ; then
  cat ${ftmp4}

  if [ ${ZABBIX_SENDER} -eq 1 ] ; then
    IPS="`cat ${ftmp4}`"
    zabbix_sender -c $ZABBIX_CONF -k "dnsbl[ips]" -o "${IPS}" 2>&1 >/dev/null
  fi
  _exit 0
fi

echo -n '' >${REPORTFILE}
while read IPLINE ; do
  echo -n '' >$ftmp1
  # extract Domain from IPLINE and cut that Domain out of IPLINE
  DOMAIN="`echo "${IPLINE}" | sed 's/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*//g;s/ *//g'`"
  SUBJECT="`echo "${DOMAIN}:" | awk '{ printf("%-25s",$0) }'`"
  [ -z "${DOMAIN}" ] || IPLINE="`echo "${IPLINE}" | sed -e "s/${DOMAIN}//"`"

  for IP in ${IPLINE} ; do
    set ${IP//./ }
    [ -z "${DOMAIN}" ] && SUBJECT="`echo "${IP}:" | awk '{ printf("%-18s\n",$0) }'`"
    while read BLDOMAINLINE ; do
      BLDOMAIN=`echo "${BLDOMAINLINE}" | awk '{print $1}'`
      ASKDNS=
      echo "${BLDOMAINLINE}" | grep -q ' 1$' && ASKDNS=${DNS_SERVER}
      if host -tA -W$TIMEOUT "$4.$3.$2.$1.${BLDOMAIN}." ${ASKDNS} | grep -q 'has address 127\.0\.0\.' ; then
        TXT="`host -t txt -W$TIMEOUT "$4.$3.$2.$1.${BLDOMAIN}." ${ASKDNS} | grep -o 'descriptive text .*' | cut -b19- | sed 's/\"$//'`"
        echo -n "${SUBJECT} ${TXT}" >>$ftmp1
        [ -z "${TXT}" ] && echo -n "listed @ ${BLDOMAIN}" >>$ftmp1
        echo '' >>$ftmp1
      fi
    done <$ftmp3
  done
  cat $ftmp1 | tee -a ${REPORTFILE}
done <$ftmp4

if [ -s ${REPORTFILE} ] ; then
  IPS="`cat ${REPORTFILE}`"
  zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "${IPS}" 2>&1 >/dev/null
else
  echo '.'
  [ ${ZABBIX_SENDER} -eq 1 ] && zabbix_sender -c $ZABBIX_CONF -k "dnsbl[chk]" -o "." 2>&1 >/dev/null
fi
_exit 0