Improve filtering of backtrace noise in the memcheck bucket

and update exp files accordingly. This works well for x86
and all testcases pass on my machine.
New file filter_memcheck to do the work. 
There is a bit of a ripple here as filter_memcheck requires
command line arguments to be passed in. So all users of 
filter_memcheck (direct or indirect) were updated as well.
filter_stderr was simplified as was filter_libc.


git-svn-id: svn://svn.valgrind.org/valgrind/trunk@12091

gdbserver_tests/filter_memcheck_monitor

@@ -4,7 +4,7 @@
 
 dir=`dirname $0`
 
-$dir/../memcheck/tests/filter_stderr                   |
+$dir/../memcheck/tests/filter_stderr "$@"              |
 
 # filter vgdb messages
 $dir/filter_vgdb                                       |

gdbserver_tests/mcleak.vgtest

@@ -9,3 +9,5 @@ argsB: --quiet -l 60 --nx ../memcheck/tests/leak-delta
 stdinB: mcleak.stdinB.gdb
 stdoutB_filter: filter_gdb
 stderrB_filter: filter_memcheck_monitor
+stderr_filter_args:  leak-delta.c
+stderrB_filter_args: leak-delta.c

gdbserver_tests/mcsignopass.vgtest

@@ -8,6 +8,7 @@ prereq: test -e gdb
 prog: ../none/tests/faultstatus
 vgopts: --tool=memcheck --vgdb=full --vex-iropt-precise-memory-exns=yes --vgdb-error=0 --vgdb-prefix=./vgdb-prefix-mcsignopass
 stderr_filter: filter_memcheck_monitor
+stderr_filter_args: faultstatus.c
 progB: gdb
 argsB: --quiet -l 60 --nx ../none/tests/faultstatus
 stdinB: mcsignopass.stdinB.gdb

gdbserver_tests/mcsigpass.vgtest

@@ -3,9 +3,9 @@ prereq: test -e gdb
 prog: ../none/tests/faultstatus
 vgopts: --tool=memcheck --vgdb=full --vex-iropt-precise-memory-exns=yes --vgdb-error=0 --vgdb-prefix=./vgdb-prefix-mcsigpass
 stderr_filter: filter_memcheck_monitor
+stderr_filter_args: faultstatus.c
 progB: gdb
 argsB: --quiet -l 60 --nx ../none/tests/faultstatus
 stdinB: mcsigpass.stdinB.gdb
 stdoutB_filter: filter_gdb
 stderrB_filter: filter_make_empty
-

memcheck/tests/amd64-linux/filter_defcfaexpr

@@ -25,4 +25,4 @@
 
 sed "s/\/.*\/tests\/amd64-linux\/defcfaexpr/bogus.S:0/" | \
       sed "s/(in /(/" | \
-      ./filter_stderr
+      ./filter_stderr "$@"

memcheck/tests/amd64-linux/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/amd64/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/badfree-2trace.vgtest

@@ -1,2 +1,3 @@
 vgopts: --num-callers=2 -q
 prog:   badfree
+stderr_filter_args: badfree.c

memcheck/tests/badfree3.vgtest

@@ -1,2 +1,3 @@
 prog: badfree
 vgopts: -q --fullpath-after=${PWD}/
+stderr_filter_args: badfree.c

memcheck/tests/badjump.stderr.exp

@@ -1,14 +1,12 @@
 
 Jump to the invalid address stated on the next line
-   at 0x........: ???
-   by 0x........: (below main)
+   ...
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 
 Process terminating with default action of signal 11 (SIGSEGV)
  Access not within mapped region at address 0x........
-   at 0x........: ???
-   by 0x........: (below main)
+   ...
  If you believe this happened as a result of a stack
  overflow in your program's main thread (unlikely but
  possible), you can try to increase the size of the

memcheck/tests/badjump.stderr.exp-s390x

@@ -1,13 +1,13 @@
 
 Jump to the invalid address stated on the next line
-   at 0x........: ???
+   ...
    by 0x........: main (badjump.c:17)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 
 Process terminating with default action of signal 11 (SIGSEGV)
  Access not within mapped region at address 0x........
-   at 0x........: ???
+   ...
    by 0x........: main (badjump.c:17)
  If you believe this happened as a result of a stack
  overflow in your program's main thread (unlikely but

memcheck/tests/badjump2.stderr.exp

@@ -1,6 +1,5 @@
 Jump to the invalid address stated on the next line
-   at 0x........: ???
-   by 0x........: (below main)
+   ...
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 Signal caught, as expected

memcheck/tests/badjump2.stderr.exp-s390x

@@ -1,5 +1,5 @@
 Jump to the invalid address stated on the next line
-   at 0x........: ???
+   ...
    by 0x........: main (badjump2.c:46)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 

memcheck/tests/badpoll.stderr.exp

@@ -1,11 +1,13 @@
 Syscall param poll(ufds.fd) points to uninitialised byte(s)
    ...
+   by 0x........: main (badpoll.c:22)
  Address 0x........ is 8 bytes inside a block of size 15 alloc'd
    at 0x........: malloc (vg_replace_malloc.c:...)
    by 0x........: main (badpoll.c:12)
 
 Syscall param poll(ufds.reventss) points to unaddressable byte(s)
    ...
+   by 0x........: main (badpoll.c:22)
  Address 0x........ is 0 bytes after a block of size 15 alloc'd
    at 0x........: malloc (vg_replace_malloc.c:...)
    by 0x........: main (badpoll.c:12)

memcheck/tests/buflen_check.stderr.exp

@@ -1,9 +1,11 @@
 Syscall param socketcall.getsockname(name) points to unaddressable byte(s)
    ...
+   by 0x........: main (buflen_check.c:18)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 Syscall param socketcall.getsockname(namelen_in) points to unaddressable byte(s)
    ...
+   by 0x........: main (buflen_check.c:19)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 getsockname(1) failed

memcheck/tests/buflen_check.stderr.exp-kfail

@@ -0,0 +1,10 @@
+Syscall param socketcall.getsockname(name) points to unaddressable byte(s)
+   ...
+ Address 0x........ is not stack'd, malloc'd or (recently) free'd
+
+Syscall param socketcall.getsockname(namelen_in) points to unaddressable byte(s)
+   ...
+ Address 0x........ is not stack'd, malloc'd or (recently) free'd
+
+getsockname(1) failed
+getsockname(2) failed

memcheck/tests/darwin/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/err_disable3.stderr.exp

@@ -11,7 +11,7 @@ Thread 2:
 Invalid read of size 1
    at 0x........: err (err_disable3.c:25)
    by 0x........: child_fn (err_disable3.c:31)
-   by 0x........: start_thread (pthread_create.c:304)
+   ...
  Address 0x........ is 5 bytes inside a block of size 10 free'd
    at 0x........: free (vg_replace_malloc.c:...)
    by 0x........: main (err_disable3.c:42)

memcheck/tests/err_disable3.vgtest

@@ -1,2 +1,2 @@
 prog: err_disable3
-vgopts: -q --num-callers=3
+vgopts: -q

memcheck/tests/filter_addressable

@@ -1,6 +1,6 @@
 #! /bin/sh
 
-./filter_allocs |
+./filter_allocs "$@" |
 perl -p -e 's/(default action of signal) [0-9]+ \(SIG(SEGV|BUS)\)/$1 N \(SIGSEGV or SIGBUS\)/' |
 perl -p -e 's/(Bad permissions for mapped region|Access not within mapped region|Non-existent physical address) at address 0x/Bad memory (SIGSEGV or SIGBUS) at address 0x/'
 

memcheck/tests/filter_allocs

@@ -1,6 +1,6 @@
 #! /bin/sh
 
-./filter_stderr |
+./filter_stderr "$@" |
 sed -e "s/in use at exit: [0-9,]* bytes in [0-9,]* blocks/in use at exit: ... bytes in ... blocks/" \
     -e  "s/total heap usage: [0-9,]* allocs, [0-9,]* frees, [0-9,]* bytes allocated/total heap usage: ... allocs, ... frees, ... bytes allocated/"
 

memcheck/tests/filter_leak_cases_possible

@@ -1,4 +1,4 @@
 #! /bin/sh
 
-./filter_stderr |
+./filter_stderr "$@" |
 sed -e 's/^leaked.*$//' -e 's/^dubious.*$//' -e 's/^reachable.*$//' -e 's/^suppressed:.*$//'

memcheck/tests/filter_memcheck

@@ -0,0 +1,67 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+#---------------------------------------------------------------------
+# A list of files specific to the tool at hand. Line numbers in
+# these files will be removed from backtrace entries matching these files.
+#---------------------------------------------------------------------
+my @tool_files = ( "mc_replace_strmem.c", "vg_replace_malloc.c" );
+
+
+sub massage_backtrace_line ($$$) {
+    my ($line, $tool_files, $cmdlin_files) = @_;
+    my ($string, $qstring);
+
+# If LINE matches any of the file names passed on the command line
+# (i.e. in CMDLIN_FILES) return LINE unmodified.
+
+    foreach $string (@$cmdlin_files) {
+        $qstring = quotemeta($string);
+        return $line if ($line =~ /$qstring/);
+    }
+
+# If LINE matches any of the file names in TOOL_FILES remove the line
+# number and return the so modified line.
+
+    foreach $string (@$tool_files) {
+        $qstring = quotemeta($string);
+        return $line if ($line =~ s/$qstring:[0-9]+/$string:.../m);
+# Special case for functions whose line numbers have been removed in 
+# filter_stderr_basic. FIXME: filter_stderr_basic should not do that.
+        return $line if ($line =~ s/$qstring:\.\.\./$string:.../m);
+    }
+
+# Did not match anything
+    $line =~ s/[\w]+.*/.../m;
+
+    return "$line";
+}
+
+
+#---------------------------------------------------------------------
+# Process lines. Two categories
+# (a) lines from back traces
+#     pass through those lines that contain file names we're interested in
+# (b) everything else
+#     pass through as is
+#---------------------------------------------------------------------
+my $prev_line = "";
+while (<STDIN>) {
+    my $line = $_;
+    chomp($line);
+    if ($line =~ /^\s+(at |by )/)  {   # lines in a back trace
+        $line = massage_backtrace_line($line, \@tool_files, \@ARGV);
+        if ($line =~ /\s+\.\.\./) {
+            print "$line\n" if ($prev_line !~ /\s+\.\.\./);
+        } else {
+            print "$line\n";
+        }
+    } else {
+        print "$line\n";  # everything else
+    }
+    $prev_line = $line
+}
+
+exit 0;

memcheck/tests/filter_stderr

@@ -10,31 +10,18 @@ $dir/../../tests/filter_addresses                       |
 # Remove "Memcheck, ..." line and the following copyright line.
 sed "/^Memcheck, a memory error detector/ , /./ d" |
 
-# Anonymise line numbers in mc_replace_strmem.c
-sed "s/mc_replace_strmem.c:[0-9]*/mc_replace_strmem.c:.../"  |
-
 # Replace exit_group() with exit(), because you can get either on Linux
 # depending on the system.
 perl -p -e "s/param exit_group\(status\)/param exit(status)/" |
 
-# Stack traces ending in syscalls often cause problems, with lots of libc
-# variations: different numbers of entries, line numbers or not depending on
-# whether libc debug info is present, 'writev' vs 'do_writev', 'main' vs.
-# '(below main)', etc.  
-#
-# So we just cut out the entire stack trace.  It's coarse but effective, and
-# there aren't that many cases like this, and the error line gives the
-# syscall name.  
-#
-# The '<' is to avoid matching the XML case;  we don't use '>' because that
-# appears in some of the syscall parameter names (eg "act->sa_handler").
-perl -p -0 -e 's/(Syscall param[^\n<]*\n)(   (at|by)[^\n]*\n)+/$1   ...\n/gs' |
-
 # Leak check filtering.
 sed "s/checked [0-9,]* bytes./checked ... bytes./" |
 
 # More leak check filtering.  For systems that do extra libc allocations
 # (eg. Darwin) there may be extra (reachable, and thus not shown) loss
 # records.  So we filter out the loss record numbers.
-perl -p -e "s/in loss record \d+ of \d+/in loss record ... of .../" 
+perl -p -e "s/in loss record \d+ of \d+/in loss record ... of .../" |
 
+$dir/../../memcheck/tests/filter_memcheck "$@"
+
+exit 0

memcheck/tests/filter_varinfo3

@@ -2,6 +2,6 @@
 
 # See comment at top of varinfo3.c for why these hacks are necessary
 
-./filter_stderr |
+./filter_stderr "$@" |
 sed "s/static_local_def\.[0-9]*/static_local_def\.XXXX/g" |
 sed "s/static_local_undef\.[0-9]*/static_local_undef\.XXXX/g"

memcheck/tests/filter_xml

@@ -1,6 +1,6 @@
 #! /bin/sh
 
-./filter_stderr |
+./filter_stderr "$@" |
 sed "s/<tid>[0-9]*<\/tid>/<tid>...<\/tid>/" |
 sed "s/<pid>[0-9]*<\/pid>/<pid>...<\/pid>/" |
 sed "s/<ppid>[0-9]*<\/ppid>/<ppid>...<\/ppid>/" |

memcheck/tests/leak-cases-full.vgtest

@@ -1,2 +1,3 @@
 prog: leak-cases
 vgopts: -q --leak-check=full --leak-resolution=high
+stderr_filter_args: leak-cases.c

memcheck/tests/leak-cases-possible.vgtest

@@ -1,3 +1,4 @@
 prog: leak-cases
 vgopts: -q --leak-check=full --leak-resolution=high --show-possibly-lost=no
 stderr_filter: filter_leak_cases_possible
+stderr_filter_args: leak-cases.c

memcheck/tests/linux/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/linux/with-space.vgtest

@@ -12,3 +12,4 @@ prereq: cp ../manuel1 'with space'
 prog: 'with space'
 vgopts: -q
 cleanup: rm 'with space'
+stderr_filter_args: manuel1.c

memcheck/tests/partial_load_dflt.vgtest

@@ -1,2 +1,3 @@
 prog: partial_load
 stderr_filter: filter_allocs
+stderr_filter_args: partial_load.c

memcheck/tests/partial_load_ok.vgtest

@@ -1,3 +1,4 @@
 prog: partial_load
 vgopts: --partial-loads-ok=yes
 stderr_filter: filter_allocs
+stderr_filter_args: partial_load.c

memcheck/tests/ppc32/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/ppc64/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/supp_unknown.stderr.exp

@@ -1,8 +1,7 @@
 
 Process terminating with default action of signal 11 (SIGSEGV)
  Access not within mapped region at address 0x........
-   at 0x........: ???
-   by 0x........: (below main)
+   ...
  If you believe this happened as a result of a stack
  overflow in your program's main thread (unlikely but
  possible), you can try to increase the size of the

memcheck/tests/writev.stderr.exp

@@ -1,16 +1,19 @@
 Test file created.
 Syscall param writev(vector[...]) points to unaddressable byte(s)
-   ...
+   at 0x........: writev (writev.c:51)
+   by 0x........: main (writev.c:56)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 Received EFAULT as expected
 Syscall param writev(vector) points to unaddressable byte(s)
-   ...
+   at 0x........: writev (writev.c:51)
+   by 0x........: main (writev.c:68)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 Received EINVAL as expected
 Syscall param readv(vector) points to unaddressable byte(s)
    ...
+   by 0x........: main (writev.c:76)
  Address 0x........ is not stack'd, malloc'd or (recently) free'd
 
 Received EINVAL as expected

memcheck/tests/x86-linux/filter_scalar_exit_group

@@ -6,7 +6,6 @@
 
 perl -p -e "s/exit_group/EXIT_GROUP/" |
 
-../filter_stderr |
+../filter_stderr "$@" |
 
 perl -p -e "s/EXIT_GROUP/exit_group/"
-

memcheck/tests/x86-linux/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/x86-linux/scalar_supp.stderr.exp

@@ -1,9 +1,12 @@
 Syscall param (syscallno) contains uninitialised byte(s)
    ...
+   by 0x........: main (scalar_supp.c:18)
 
 Syscall param write(fd) contains uninitialised byte(s)
    ...
+   by 0x........: main (scalar_supp.c:18)
 
 Syscall param write(count) contains uninitialised byte(s)
    ...
+   by 0x........: main (scalar_supp.c:18)
 

memcheck/tests/x86/filter_pushfpopf

@@ -1,5 +1,5 @@
 #! /bin/sh
 
-./filter_stderr |
+./filter_stderr "$@" |
 sed "s/: fooble ([^)]*)/: fooble (...)/"
 

memcheck/tests/x86/filter_stderr

@@ -1,3 +1,3 @@
 #! /bin/sh
 
-../filter_stderr
+../filter_stderr "$@"

memcheck/tests/x86/filter_tronical

@@ -1,6 +1,6 @@
 #! /bin/sh
 
-./filter_stderr | \
+./filter_stderr "$@" | \
 sed "s/: get ([^)]*)/: get (...)/" |
 sed "s/: main ([^)]*)/: main (...)/"
 

tests/filter_libc

@@ -2,12 +2,6 @@
 
 use strict;
 
-my @libc_symbols = qw(__libc_start_main accept execve fcntl
-                      getsockname poll readv recvmsg
-                      socket socketpair syscall writev);
-
-my $libc_symbols = join("|", @libc_symbols);
-
 while (<>)
 {
     s/ __getsockname / getsockname /;
@@ -23,8 +17,6 @@ while (<>)
     # libc, on some (eg. Darwin) it will be in the main executable.
     s/\(below main\) \(.+\)$/(below main)/;
 
-    s/($libc_symbols) \(.+\.[cS]:\d+\)$/$1 (in \/...libc...)/;
-
     # Merge the different C++ operator variations.
     s/(at.*)__builtin_new/$1...operator new.../;
     s/(at.*)operator new\(unsigned(| int| long)\)/$1...operator new.../;
@@ -38,10 +30,6 @@ while (<>)
     s/(at.*)__builtin_vec_delete/$1...operator delete[].../;
     s/(at.*)operator delete\[\]\(void\*\)/$1...operator delete[].../;
 
-    # Tidy up in cases where glibc (+ libdl + libpthread + ld) have
-    # been built with debugging information, hence source locs are present.
-    s/\((exit|_exit|brk|sbrk).c:[0-9]*\)/(in \/...libc...)/;
-
     print;
 }