#!/usr/bin/perl # # 6/22/2004 Author: Joe Barbish, I bequeath this perl script to public domain. # It can be copied and distributed for free by anyone to anyone by any manner. # # Description: This perl script is an official DShield client who's # purpose is to read your FreeBSD ipfilter firewall ipmon log file and # convert the log records to the standard DShield reporting record format, # and imbed the converted records into the body of an email that gets # sent to DShield for automatic addition to their database and abuse reporting # to the offenders ISP if you are an subscribed DShield member. # Script contains user customable defaults which can be overridden with # command line flags. # # Last verification test June 22 2004 using FreeBSD Release 4.9 & 4.10 # ############################################################################## # # DShield report format is a tab delimited format containing the following # items (in this order): # 1.date (YYYY-MM-DD HH:MM:SS tzHH:tzMM). tz means timezone from GMT # 2.Dshield user # (0 if not subscribed DShield member) # 3.count (number, used to summarize identical records, default=1) # 4.source IP address (in 0.0.0.0 format) # 5.source port # 6.target IP address (in 0.0.0.0 format) # 7.target port # 8.protocol (either number or text like 'TCP','UDP'...) # 9.TCP flags (S-SYN, A-ACK, F-FIN, U-URG, R-RST, P-PSH) ############################################################################## # Note1 Script must have execute permission chmod 760 dshield-freebsd.pl # Note2 Be careful if you edit ipfilter log file. This script expects all # log records to end with one blank and no blank records in log file. # Marker offset file will get hosed if log file ends with blank record # and not be able to position correctly on next run. Check log file # if you have positioning problems. When you look at marker file contents # first record is the number of positions indexed into file as starting # position of last log record processed. The second record is an image # copy of the last log record processed. # # Note3 The perl Net::Netmask function used below is not installed in the # perl version that is delivered as part of the standard FreeBSD # sysinstall process. # # pkg_add -rv p5-Net-Netmask will install it. # # (note capital letter N in package name so you spell it correctly) # After pkg_add completes, follow with rehash command to enable it. # # There is also an cvsup port version of p5-Net-Netmask or you can use # the perl cpan method to add the module. use Net::Netmask; use POSIX (strftime); use Getopt::Std; use IO::File; getopts("h:f:t:c:u:l:m:e:v:s:"); $verbose=$opt_v; $version="DShield-FreeBSD-IPFilter v20040622"; if ($opt_h) { print <<_EOF_; * $version, Joe Barbish update v20040622 Usage: dshield-freebsd [ -h Display help info ] [ -f email_from ] [ -t email_to ] [ -c email_cc ] [ -u Dshield userid ] [ -l /path/ipflogfile ] [ -m log_offset_file ] [ -e exclusion_list_file ] [ -s sendmail_location ] [ -v1 turns on verbose debug ] _EOF_ exit } # # User customable script default Parameter definitions # # DShield assigned user number (0 if not subscribed) $userid=0; # Sender email address (don't forget backslash before @) # If subscribed to DShield this email address must # match the email address you used to subscribe. #$email_from="root\@this.machine"; # The To email address to send your ipfilter converted log records # to as content in the body of the email. (don't forget backslash before @) # Testing send it to your self so you can verify things are working. #$email_to="root\@this.machine"; # live Production #$email_to="report\@dshield.org"; # The CC email address (don't forget backslash before @) # Send copy of email to your self to verify and track regular submissions. #$email_cc="root\@this.machine"; $email_cc=""; # Path and file name of your ipfilter log file #$logfile="/var/log/ipf.log"; #$logfile="/var/log/security.0"; # Path and file name of your marker offset file. # Used to know where last dshield-freebsd run stopped in the log. # One will be created if one does not exist. #$marker="/root/bin/abuse.dshield.offset.marker"; # Exclusion list file (regex lines) # Used to exclude ipfilter log records containing LAN IP address # or any other log records you want from # being processed and sent to DShield. # DShield considers all reserved private Lan IP address ranges as invalid. # # If you have no exclusion file all the records in the ipfilter # log file will be processed and emailed. # # DShield report exclusion list file command syntax. # # Format: # Type=IP # # with type: # SI (Source IP) # TI (Target IP) # SP (Source Port) # TP (Target port) # port=single value between 1-65535 # and IP: # 216.240.32.0/24 The preferred form. # 216.240.32.0:255.255.255.0 # 216.240.32.0-255.255.255.0 # 216.240.32.0', '255.255.255.0 # 216.240.32.0', '0xffffff00 # 216.240.32.0 - 216.240.32.255 # 216.240.32.4 A /32 block. # 216.240.32 Always a /24 block. # 216.240 Always a /16 block. # 140 Always a /8 block. # 216.240.32/24 # 216.240/16 # # Example file content: #SI=0.0.0.0 #TI=255.255.255.255 #SI=192.168.0.0/24 #TI=192.168.0.0/24 #SI=192.168.1.0/24 #TI=192.168.1.0/24 #SP=4661 #TP=4661 #TP=4662 # Path and file name of your exclusion file. #$exclusions="/etc/dshield.excluded.lst"; $exclusions="/root/bin/abuse.dshield.excluded.lst"; # Pattern to match ipmon lines. # Change tun0 to your firewall external interface name #$logpattern="ipmon.*tun0"; $logpattern="ipmon.*rl0"; # Default path to FreeBSD install delivered sendmail. # Option to change path in case you installed newer version. $sendmail="/usr/sbin/sendmail"; ############################################################################## ### From here, no changes are required ####################################### ############################################################################## # DShield requires all submitted firewall logs include the timezone so the # records can be synchronized for an world wide view. # This gets your system time zone which you set during sysinstall # or with tzsetup command. ($timezone = strftime("%z", localtime)) =~ s/(\d\d)(\d\d)/$1:$2/; debug("Checking for flag override arguments\n"); # Command line flags to over ride script default arguments if ($opt_f ne "") { $email_from=$opt_f; } if ($opt_t ne "") { $email_to=$opt_t; } if ($opt_c ne "") { $email_cc=$opt_c; } if ($opt_u ne "") { $userid=$opt_u; } if ($opt_l ne "") { $logfile=$opt_l; } if ($opt_m ne "") { $marker=$opt_m; } if ($opt_e ne "") { $exclusions=$opt_e; } if ($opt_s ne "") { $sendmail=$opt_s; } # Create run history file system("date >> /var/log/abuse.dshield.run.history"); # Read marker offset file & save positioning info debug("Loading offset file\n"); open(IN,$marker); $offset_line=; # offset number of text positions into file $offset_value=; # copy of last complete log line processed chop($offset_line); chop($offset_value); close(IN); debug("offset_line = $offset_line\n"); debug("offset_value = $offset_value\n"); # Read excluded file & save info debug("Loading exclusion file\n"); @excluded=(); open(IN,$exclusions); while () { chop; s/^\s+//; if ($_ eq "") { next; } if ($_ !~ /^#/) { push(@excluded,$_); } } close(IN); debug("Opening log file\n"); $fd = new IO::File; $fd->open($logfile,"r") || die "Cant open log file"; # position offset if ($offset_line != 0) # not empty or absent offset file { debug("Positioning offset ($offset_line)\n"); $fd->seek($offset_line,0); # set to old pointer value $line=<$fd>; chop($line); debug("Comparing log record: $line\n"); debug("To positioning record: $offset_value\n"); if (($line ne $offset_value) && ($offset_value ne "")) # position is incorrect, line differs, set to 0 { debug("Line differs, positioning to 0\n"); $fd->seek(0,0); } else { debug("Line positioning matches\n"); } } if ($fd->eof()) { $fd->close(); close(OUT); print "This log file has been previously processed.\n"; exit; } # Prepare sendmail debug("Piping to sendmail\n"); if ($verbose==1) { open(OUT,"| /bin/cat") || die "Cant open sendmail pipe"; } else { open(OUT,"| $sendmail -t -oi") || die "Cant open sendmail pipe"; } print(OUT "From: $email_from\n"); print(OUT "To: $email_to\n"); print(OUT "Cc: $email_cc\n"); print(OUT "Subject: FORMAT DSHIELD USERID $userid TZ $timezone ($version)\n\n"); $curpos=$fd->tell(); debug("File pointer located at $curpos, reading logs\n"); # init some control fields $first_time = "YES"; $yes = "YES"; $prev_log_line_count=0; # What year are we?? $year=1900+(localtime(time)) [5]; # Starting roll through log, building DShield record in email body while (1) { # Log line format is: # Aug 14 22:49:36 hostname ipmon[xx]: 22:49:35.901496 tun0 @0:16 b 200.60.255.45,4123 -> 80.13.151.54,135 PR tcp len 20 48 -S IN $line_curpos=$fd->tell(); $line=<$fd>; chop($line); debug("log record = $line\n"); # parse line to extract relevant fields @f=split(/\s+/,$line); # re-init work fields to null $log_line_date=""; $log_line_time=""; $log_line_ip_src=""; $log_line_ip_trg=""; $log_line_port_src=""; $log_line_port_trg=""; $log_line_proto=""; $log_line_flags=""; $log_line_author=$userid; $log_line_dupnum=""; $log_line_dupsufx=""; $month=convmonth($f[0]); $log_line_date=sprintf("%04d-%02d-%02d",$year,$month,$f[1]); $log_line_time=$f[2]; # For ipmom log records that have dup counter field the parse line # fields end up having different field displacement. $dupctr=$f[6]; debug("Log line dup counter = $dupctr\n"); $log_line_dupnum = substr($dupctr, 0, length($dupctr) -1); $log_line_dupsufx = substr($dupctr, -1, 1); debug("counter = $log_line_dupnum\n"); debug("counter = $log_line_dupsufx\n"); if ($log_line_dupsufx eq "x") { debug("this log rec has dup counter\n"); @g=split(/,/,$f[10]); $log_line_ip_src=$g[0]; $log_line_port_src=$g[1]; @g=split(/,/,$f[12]); $log_line_ip_trg=$g[0]; $log_line_port_trg=$g[1]; $_=$f[14]; tr/a-z/A-Z/; $log_line_proto=$_; if ($log_line_proto =~ /TCP/i) { $_=$f[18]; s/-//; $log_line_flags=$_; if ($log_line_flags =~ /frag/i) { $log_line_flags=""; } } } else { debug("this log rec does not have dup counter\n"); @g=split(/,/,$f[9]); $log_line_ip_src=$g[0]; $log_line_port_src=$g[1]; @g=split(/,/,$f[11]); $log_line_ip_trg=$g[0]; $log_line_port_trg=$g[1]; $_=$f[13]; tr/a-z/A-Z/; $log_line_proto=$_; if ($log_line_proto =~ /TCP/i) { $_=$f[17]; s/-//; $log_line_flags=$_; if ($log_line_flags =~ /frag/i) { $log_line_flags=""; } } } if ($first_time eq $yes) { debug("first time logic $first_time\n"); # populate summarize logic control fields one time at start $prev_log_line_date=$log_line_date; $prev_log_line_time=$log_line_time; $prev_log_line_ip_src=$log_line_ip_src; $prev_log_line_port_src=$log_line_port_src; $prev_log_line_ip_trg=$log_line_ip_trg; $prev_log_line_port_trg=$log_line_port_trg; $prev_log_line_proto=$log_line_proto; $prev_log_line_flags=$log_line_flags; $first_time = "NO"; debug("first time logic $first_time\n"); } # logic to drop excluded log records based on exclusion file if (($line =~ /$log_pattern/) && ($log_line_ip_src =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) && ($log_line_ip_trg =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) && (NotExcluded("SI",$log_line_ip_src)) && (NotExcluded("SP",$log_line_port_src)) && (NotExcluded("TI",$log_line_ip_trg)) && (NotExcluded("TP",$log_line_port_trg))) { # logic to summarize dup log records in same seconds if (($prev_log_line_date eq $log_line_date) && ($prev_log_line_time eq $log_line_time) && ($prev_log_line_ip_src eq $log_line_ip_src) && ($prev_log_line_port_src eq $log_line_port_src) && ($prev_log_line_ip_trg eq $log_line_ip_trg) && ($prev_log_line_port_trg eq $log_line_port_trg) && ($prev_log_line_proto eq $log_line_proto) && ($prev_log_line_flags eq $log_line_flags)) { $prev_log_line_count++; debug("prev = current summarize logic $prev_log_line_count\n");} else { #load email body with dshield record format containing log contents debug("load email body line with log line contents\n"); print(OUT "$prev_log_line_date $prev_log_line_time $timezone\t"); print(OUT "$log_line_author\t$prev_log_line_count\t"); print(OUT "$prev_log_line_ip_src\t$prev_log_line_port_src\t"); print(OUT "$prev_log_line_ip_trg\t$prev_log_line_port_trg\t"); print(OUT "$prev_log_line_proto\t$prev_log_line_flags\n"); # populate summarize logic control fields $prev_log_line_date=$log_line_date; $prev_log_line_time=$log_line_time; $prev_log_line_ip_src=$log_line_ip_src; $prev_log_line_port_src=$log_line_port_src; $prev_log_line_ip_trg=$log_line_ip_trg; $prev_log_line_port_trg=$log_line_port_trg; $prev_log_line_proto=$log_line_proto; $prev_log_line_flags=$log_line_flags; $prev_log_line_count=1; if ($log_line_dupsufx eq "x") {$prev_log_line_count = $log_line_dupnum}; } } if ($fd->eof()) { debug("EOF load email body line with log line contents\n"); print(OUT "$log_line_date $log_line_time $timezone\t"); print(OUT "$log_line_author\t$prev_log_line_count\t"); print(OUT "$log_line_ip_src\t$log_line_port_src\t"); print(OUT "$log_line_ip_trg\t$log_line_port_trg\t"); print(OUT "$log_line_proto\t$log_line_flags\n"); last; } } $fd->close(); close(OUT); # at this point all the ipfilter log records have been processed # and the log file has been closed. # Now write the positioning offset info to the marker file. open(OUT,">$marker"); print(OUT "$line_curpos\n"); print(OUT "$line\n"); close(OUT); exit; ############# every thing beyond this point are subroutines ############ sub convmonth() { my %months_tab = (Jan=>1, Feb=>2, Mar=>3, Apr=>4, May=>5, Jun=>6, Jul=>7, Aug=>8, Sep=>9, Oct=>10, Nov=>11, Dec=>12); return $months_tab{$_[0]}; } sub NotExcluded { my ($excl_type)=@_[0]; my ($tocheck)=@_[1]; my ($i); my (@f); my ($t); for ($i=0;$i<@excluded;$i++) { @f=split(/=/,$excluded[$i]); $type=$f[0]; $value=$f[1]; if (($excl_type =~ /^SI$/i) || ($excl_type =~ /^TI$/)) { $t=new Net::Netmask($value); if (($t->match($tocheck)) && ($excl_type eq $type)) { return(0); } } elsif (($excl_type =~ /^SP$/i) || ($excl_type =~ /^TP$/)) { if (($value==$tocheck) && ($excl_type eq $type)) { return(0); } } } return(1); } sub debug { if ($verbose==1) { print(STDERR @_); } }