Skip to content

8371647: 7 Integer overflows in mlib_malloc of mlib_sys.c:85 #28560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
DamonGuy wants to merge 1 commit into
base: master
Choose a base branch
from
+42 −0
Draft

8371647: 7 Integer overflows in mlib_malloc of mlib_sys.c:85 #28560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/java.desktop/share/native/libmlib_image/mlib_ImageConvMxN_Fp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
#include "mlib_ImageCheck.h"
#include "mlib_SysMath.h"
#include "mlib_ImageConv.h"
#include "safe_math.h"

/***************************************************************/
static void mlib_ImageConvMxNMulAdd_F32(mlib_f32 *dst,
Expand Down Expand Up @@ -272,6 +273,13 @@ mlib_status mlib_convMxNext_f32(mlib_image *dst,
mlib_s32 nch = mlib_ImageGetChannels(dst);
mlib_s32 i, j, j1, k;

if (!SAFE_TO_ADD(wid_e, m) ||
!SAFE_TO_MULT(3, wid_e + m) ||
!SAFE_TO_MULT((3 * wid_e + m),
(mlib_d64)sizeof(mlib_d64))) {
return MLIB_FAILURE;
}

if (3 * wid_e + m > 1024) {
dsa = mlib_malloc((3 * wid_e + m) * sizeof(mlib_d64));

Expand Down Expand Up @@ -629,6 +637,13 @@ mlib_status mlib_convMxNext_d64(mlib_image *dst,
mlib_s32 nch = mlib_ImageGetChannels(dst);
mlib_s32 i, j, j1, k;

if (!SAFE_TO_ADD(wid_e, m) ||
!SAFE_TO_MULT(3, wid_e + m) ||
!SAFE_TO_MULT((3 * wid_e + m),
(mlib_d64)sizeof(mlib_d64))) {
return MLIB_FAILURE;
}

if (3 * wid_e + m > 1024) {
dsa = mlib_malloc((3 * wid_e + m) * sizeof(mlib_d64));

Expand Down
8 changes: 8 additions & 0 deletions src/java.desktop/share/native/libmlib_image/mlib_ImageConvMxN_ext.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@

#include "mlib_image.h"
#include "mlib_ImageConv.h"
#include "safe_math.h"

/***************************************************************/
static void mlib_ImageConvMxNMulAdd_S32(mlib_d64 *dst,
Expand Down Expand Up @@ -229,6 +230,13 @@ mlib_status mlib_convMxNext_s32(mlib_image *dst,

/* internal buffer */

if (!SAFE_TO_ADD(wid_e, m) ||
!SAFE_TO_MULT(3, wid_e + m) ||
!SAFE_TO_MULT((3 * wid_e + m),
(mlib_d64)sizeof(mlib_d64))) {
return MLIB_FAILURE;
}

if (3 * wid_e + m > 1024) {
dsa = mlib_malloc((3 * wid_e + m) * sizeof(mlib_d64));

Expand Down
5 changes: 5 additions & 0 deletions src/java.desktop/share/native/libmlib_image/mlib_ImageConv_16ext.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "mlib_image.h"
#include "mlib_ImageConv.h"
#include "mlib_c_ImageConv.h"
#include "safe_math.h"

/*
* This define switches between functions of different data types
Expand Down Expand Up @@ -265,6 +266,8 @@ static mlib_status mlib_ImageConv1xN_ext(mlib_image *dst,
bsize = 2 * (smax_hsize + 1);

if (bsize > BUFF_SIZE) {
if (!SAFE_TO_MULT(bsize, (mlib_s32)sizeof(FTYPE))) return MLIB_FAILURE;

pbuff = mlib_malloc(sizeof(FTYPE)*bsize);
Comment on lines 266 to 271
Copy link
Contributor

@prrace prrace Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If mlib_malloc ends up in
void *__mlib_malloc(mlib_u32 size);
which I think it must do, because I can't find anything else,
then that accepts an unsigned 32 bit int, which makes sense because malloc accepts a size_t which is unsigned.

Note that sizeof() returns size_t too, so the multiplication result should be promoted to unsigned in the existing code, and preserved when passed as an arg.

But SAFE_TO_MULT will return a failure on overflow of signed arithmetic. So I think we need something different here so we don't reject cases which are actually OK. ie in at least cases like this, we want to detect overflow of 32 bit unsigned, not 32 bit signed.


if (pbuff == NULL) return MLIB_FAILURE;
Expand Down Expand Up @@ -495,6 +498,8 @@ mlib_status CONV_FUNC_MxN
mn = m*n;

if (mn > 256) {
if (!SAFE_TO_MULT(mn, (mlib_d64)sizeof(mlib_d64))) return MLIB_FAILURE;

k = mlib_malloc(mn*sizeof(mlib_d64));

if (k == NULL) return MLIB_FAILURE;
Expand Down
7 changes: 7 additions & 0 deletions src/java.desktop/share/native/libmlib_image/mlib_ImageConv_8ext.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "mlib_image.h"
#include "mlib_ImageConv.h"
#include "mlib_c_ImageConv.h"
#include "safe_math.h"

/*
* This define switches between functions of different data types
Expand Down Expand Up @@ -919,6 +920,12 @@ mlib_status CONV_FUNC_MxN_I
for (l = 0; l < (n + 1); l++) buffs[l + (n + 1)] = buffs[l];
buffd = buffs[n] + swid;

if (!SAFE_TO_MULT(m, n) ||
!SAFE_TO_MULT((m * n),
(mlib_s32)sizeof(mlib_s32))) {
return MLIB_FAILURE;
}

if (m*n > MAX_N*MAX_N) {
k = mlib_malloc(sizeof(mlib_s32)*(m*n));

Expand Down
7 changes: 7 additions & 0 deletions src/java.desktop/share/native/libmlib_image/mlib_ImageConv_u16ext.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "mlib_image.h"
#include "mlib_ImageConv.h"
#include "mlib_c_ImageConv.h"
#include "safe_math.h"

/*
* This define switches between functions of different data types
Expand Down Expand Up @@ -942,6 +943,12 @@ mlib_status CONV_FUNC_MxN_I
for (l = 0; l < (n + 1); l++) buffs[l + (n + 1)] = buffs[l];
buffd = buffs[n] + swid;

if (!SAFE_TO_MULT(m, n) ||
!SAFE_TO_MULT((m * n),
(mlib_s32)sizeof(mlib_s32))) {
return MLIB_FAILURE;
}

if (m*n > MAX_N*MAX_N) {
k = mlib_malloc(sizeof(mlib_s32)*(m*n));

Expand Down