diff --git a/src/proxy.cc b/src/proxy.cc
index 0d6fcd511..5155876ee 100644
--- a/src/proxy.cc
+++ b/src/proxy.cc
@@ -1931,7 +1931,10 @@ ncclResult_t ncclProxyStop(struct ncclComm* comm) {
             if (sharedProxyState->sharedDevMems[i]) {
               if (!ncclCuMemEnable()) {
                 CUDACHECK(cudaIpcCloseMemHandle(sharedProxyState->sharedDevMems[i]));
+              } else {
+                NCCLCHECK(ncclCudaFree((char*)sharedProxyState->sharedDevMems[i]));
               }
+              sharedProxyState->sharedDevMems[i] = NULL;
             }
             int type = ncclProxyMsgClose;
             (void)ncclSocketSend(sharedProxyState->peerSocks + i, &type, sizeof(int));