diff --git a/Tests/test_imagedraw.py b/Tests/test_imagedraw.py
index 790acee2a46..1e0dedef3cf 100644
--- a/Tests/test_imagedraw.py
+++ b/Tests/test_imagedraw.py
@@ -68,10 +68,20 @@ def test_sanity() -> None:
     draw.rectangle(list(range(4)))
 
 
-def test_valueerror() -> None:
+def test_new_color() -> None:
     with Image.open("Tests/images/chi.gif") as im:
         draw = ImageDraw.Draw(im)
+        assert len(im.palette.colors) == 249
+
+        # Test drawing a new color onto the palette
         draw.line((0, 0), fill=(0, 0, 0))
+        assert len(im.palette.colors) == 250
+        assert im.palette.dirty
+
+        # Test drawing another new color, now that the palette is dirty
+        draw.point((0, 0), fill=(1, 0, 0))
+        assert len(im.palette.colors) == 251
+        assert im.convert("RGB").getpixel((0, 0)) == (1, 0, 0)
 
 
 def test_mode_mismatch() -> None:
diff --git a/src/PIL/Image.py b/src/PIL/Image.py
index 9d50812eb3e..dc51860a040 100644
--- a/src/PIL/Image.py
+++ b/src/PIL/Image.py
@@ -897,7 +897,9 @@ def load(self) -> core.PixelAccess | None:
                 else:
                     self.im.putpalettealphas(self.info["transparency"])
                 self.palette.mode = "RGBA"
-            else:
+            elif self.palette.mode != mode:
+                # If the palette rawmode is different to the mode,
+                # then update the Python palette data
                 self.palette.palette = self.im.getpalette(
                     self.palette.mode, self.palette.mode
                 )