PR217695 malloc/calloc/realloc/memalign failure doesn't set errno to ENOMEM

When one of the allocation functions in vg_replace_malloc failed
they return NULL, but didn't set errno. This is slightly tricky since
errno is implementation defined and might be a macro. In the case of
glibc ernno is defined as:

  extern int *__errno_location (void) __THROW __attribute__ ((__const__));
  #define errno (*__errno_location ())

We can use the same trick as we use for __libc_freeres in
coregrind/vg_preloaded.c. Define the function as "weak". This means
it will only be defined if another library (glibc in this case)
actually provides a definition. Otherwise it will be NULL.
So we will only call it if it is defined and one of the allocation
functions failed, returned NULL.

Include a new linux only memcheck testcase, enomem.vgtest.

https://bugs.kde.org/show_bug.cgi?id=217695

NEWS

@@ -53,6 +53,7 @@ where XXXXXX is the bug number as listed below.
 
 140178  open("/proc/self/exe", ...); doesn't quite work
 140939 --track-fds reports leakage of stdout/in/err and doesn't respect -q
+217695  malloc/calloc/realloc/memalign failure doesn't set errno to ENOMEM
 345077  linux syscall execveat support (linux 3.19)
 369029  handle linux syscalls sched_getattr and sched_setattr
 n-i-bz  helgrind: If hg_cli__realloc fails, return NULL.

coregrind/m_replacemalloc/vg_replace_malloc.c

@@ -192,6 +192,17 @@ static void init(void);
    if (info.clo_trace_malloc) {        \
       VALGRIND_INTERNAL_PRINTF(format, ## args ); }
 
+/* Tries to set ERRNO to ENOMEM if possible.
+   Only implemented for glibc at the moment.
+*/
+#if defined(VGO_linux)
+extern int *__errno_location (void) __attribute__((weak));
+#define SET_ERRNO_ENOMEM if (__errno_location)        \
+      (*__errno_location ()) = VKI_ENOMEM;
+#else
+#define SET_ERRNO_ENOMEM {}
+#endif
+
 /* Below are new versions of malloc, __builtin_new, free,
    __builtin_delete, calloc, realloc, memalign, and friends.
 
@@ -246,6 +257,7 @@ static void init(void);
       \
       v = (void*)VALGRIND_NON_SIMD_CALL1( info.tl_##vg_replacement, n ); \
       MALLOC_TRACE(" = %p\n", v ); \
+      if (!v) SET_ERRNO_ENOMEM; \
       return v; \
    }
 
@@ -752,6 +764,7 @@ static void init(void);
       if (umulHW(size, nmemb) != 0) return NULL; \
       v = (void*)VALGRIND_NON_SIMD_CALL2( info.tl_calloc, nmemb, size ); \
       MALLOC_TRACE(" = %p\n", v ); \
+      if (!v) SET_ERRNO_ENOMEM; \
       return v; \
    }
 
@@ -826,6 +839,7 @@ static void init(void);
       } \
       v = (void*)VALGRIND_NON_SIMD_CALL2( info.tl_realloc, ptrV, new_size ); \
       MALLOC_TRACE(" = %p\n", v ); \
+      if (!v) SET_ERRNO_ENOMEM; \
       return v; \
    }
 
@@ -899,6 +913,7 @@ static void init(void);
       \
       v = (void*)VALGRIND_NON_SIMD_CALL2( info.tl_memalign, alignment, n ); \
       MALLOC_TRACE(" = %p\n", v ); \
+      if (!v) SET_ERRNO_ENOMEM; \
       return v; \
    }
 

memcheck/tests/linux/Makefile.am

@@ -29,7 +29,8 @@ EXTRA_DIST = \
 	getregset.stderr.exp getregset.stdout.exp \
 	sys-preadv_pwritev.vgtest sys-preadv_pwritev.stderr.exp \
 	sys-preadv2_pwritev2.vgtest sys-preadv2_pwritev2.stderr.exp \
-	sys-execveat.vgtest sys-execveat.stderr.exp sys-execveat.stdout.exp
+	sys-execveat.vgtest sys-execveat.stderr.exp sys-execveat.stdout.exp \
+	enomem.vgtest enomem.stderr.exp enomem.stdout.exp
 
 check_PROGRAMS = \
 	brk \
@@ -50,7 +51,8 @@ check_PROGRAMS = \
 	timerfd-syscall \
 	proc-auxv \
 	sys-execveat \
-	check_execveat
+	check_execveat \
+	enomem
 
 if HAVE_AT_FDCWD
 check_PROGRAMS += sys-openat

memcheck/tests/linux/enomem.c

@@ -0,0 +1,34 @@
+
+/* Test malloc, calloc, realloc and memalign set errno to ENOMEM */
+
+#include <errno.h>
+#include <limits.h>
+#include <malloc.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main ( void )
+{
+  char* small = malloc (16);
+  char* p;
+
+  errno = 0;
+  p = malloc(SSIZE_MAX);
+  if (!p && errno == ENOMEM) puts("malloc: Cannot allocate memory");
+
+  errno = 0;
+  p = calloc(1, SSIZE_MAX);
+  if (!p && errno == ENOMEM) puts("calloc: Cannot allocate memory");
+
+  errno = 0;
+  p = realloc(small, SSIZE_MAX);
+  if (!p && errno == ENOMEM) puts("realloc: Cannot allocate memory");
+
+  errno = 0;
+  p = memalign(64, SSIZE_MAX);
+  if (!p && errno == ENOMEM) puts("memalign: Cannot allocate memory");
+
+  free(small);
+
+  return 0;
+}

memcheck/tests/linux/enomem.stdout.exp

@@ -0,0 +1,4 @@
+malloc: Cannot allocate memory
+calloc: Cannot allocate memory
+realloc: Cannot allocate memory
+memalign: Cannot allocate memory

memcheck/tests/linux/enomem.vgtest

@@ -0,0 +1,2 @@
+prog: enomem
+vgopts: -q