[feat] fixes CoverageTool memory leaks

- adds destructors for classes and respective calls for them
- provides a certain background for implementing test project collection in the future

Author: DanielELog

VSharp.CoverageInstrumenter/profiler/ILRewriter.cpp

@@ -6,12 +6,13 @@
 #include "corprof.h"
 #include <stdexcept>
 #include "probes.h"
+#include "memory.h"
 
 #undef IfFailRet
-#define IfFailRet(EXPR) do { HRESULT hr = (EXPR); if(FAILED(hr)) { return (hr); } } while (0)
+#define IfFailRet(EXPR) do { if (std::atomic_load(&vsharp::shutdownInOrder)) { return S_OK; } HRESULT hr = (EXPR); if(FAILED(hr)) { return (hr); } } while (0)
 
 #undef IfNullRet
-#define IfNullRet(EXPR) do { if ((EXPR) == NULL) return E_OUTOFMEMORY; } while (0)
+#define IfNullRet(EXPR) do { if (std::atomic_load(&vsharp::shutdownInOrder)) { return S_OK; } if ((EXPR) == NULL) return E_OUTOFMEMORY; } while (0)
 
 struct ILInstr
 {

VSharp.CoverageInstrumenter/profiler/ILRewriter.h

@@ -8,6 +8,7 @@
 #include <vector>
 
 using namespace vsharp;
+
 extern "C" void SetEntryMain(char* assemblyName, int assemblyNameLength, char* moduleName, int moduleNameLength, int methodToken) {
     profilerState->setEntryMain(assemblyName, assemblyNameLength, moduleName, moduleNameLength, methodToken);
     LOG(tout << "received entry main" << std::endl);
@@ -18,9 +19,9 @@ extern "C" void GetHistory(UINT_PTR size, UINT_PTR bytes) {
 
     std::atomic_fetch_add(&shutdownBlockingRequestsCount, 1);
     size_t tmpSize;
-    auto tmpBytes = profilerState->coverageTracker->serializeCoverageReport(&tmpSize);
+    historyBuffer = profilerState->coverageTracker->serializeCoverageReport(&tmpSize);
     *(ULONG*)size = tmpSize;
-    *(char**)bytes = tmpBytes;
+    *(char**)bytes = historyBuffer;
 
     profilerState->coverageTracker->clear();
     profilerState->threadTracker->clear();
@@ -29,6 +30,11 @@ extern "C" void GetHistory(UINT_PTR size, UINT_PTR bytes) {
     LOG(tout << "GetHistory request handled!");
 }
 
+extern "C" void ClearHistory() {
+    delete historyBuffer;
+    historyBuffer = nullptr;
+}
+
 extern "C" void SetCurrentThreadId(int mapId) {
     LOG(tout << "Map current thread to: " << mapId);
     vsharp::profilerState->threadTracker->mapCurrentThread(mapId);

VSharp.CoverageInstrumenter/profiler/api.cpp

@@ -12,5 +12,8 @@
 extern "C" IMAGEHANDLER_API void SetEntryMain(char* assemblyName, int assemblyNameLength, char* moduleName, int moduleNameLength, int methodToken);
 extern "C" IMAGEHANDLER_API void GetHistory(UINT_PTR size, UINT_PTR bytes);
 extern "C" IMAGEHANDLER_API void SetCurrentThreadId(int mapId);
+extern "C" IMAGEHANDLER_API void ClearHistory();
+
+static char* historyBuffer = nullptr;
 
 #endif //VSHARP_COVERAGEINSTRUMENTER_API_H

VSharp.CoverageInstrumenter/profiler/api.h

@@ -31,6 +31,9 @@ CorProfiler::~CorProfiler()
 
 HRESULT STDMETHODCALLTYPE CorProfiler::Initialize(IUnknown *pICorProfilerInfoUnk)
 {
+    setbuf(stdout, NULL);
+
+    printf("PROFILER INITIALIZATION\n");
     const char* waitDebuggerAttached = std::getenv("COVERAGE_TOOL_WAIT_DEBUGGER_ATTACHED");
     volatile int done = waitDebuggerAttached == nullptr ? 1 : 0;
     while (!done) OS::sleepSeconds(1);
@@ -53,7 +56,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::Initialize(IUnknown *pICorProfilerInfoUnk
 
     // TMP Windows fix
     #undef IfFailRet
-    #define IfFailRet(EXPR) do { HRESULT hr = (EXPR); if(FAILED(hr)) { return (hr); } } while (0)
+    #define IfFailRet(EXPR) do { if (std::atomic_load(&shutdownInOrder)) return S_OK; HRESULT hr = (EXPR); if(FAILED(hr)) { return (hr); } } while (0)
     IfFailRet(this->corProfilerInfo->SetEventMask(eventMask));
 
     profilerState = new ProfilerState((ICorProfilerInfo8*)this);
@@ -64,27 +67,36 @@ HRESULT STDMETHODCALLTYPE CorProfiler::Initialize(IUnknown *pICorProfilerInfoUnk
 
 HRESULT STDMETHODCALLTYPE CorProfiler::Shutdown()
 {
-    profilerState->isFinished = true;
+    printf("PROFILER SHUTDOWN\n");
+    std::atomic_store(&shutdownInOrder, true);
 
     // waiting until all current requests are resolved
     while (std::atomic_load(&shutdownBlockingRequestsCount) > 0) {}
 
     LOG(tout << "SHUTDOWN");
     if (profilerState->isPassiveRun) {
+        printf("serializing information\n");
 
         size_t tmpSize;
-        auto tmpBytes = profilerState->coverageTracker->serializeCoverageReport(&tmpSize);;
+        auto tmpBytes = profilerState->coverageTracker->serializeCoverageReport(&tmpSize);
+
+        printf("saving to %s\n", profilerState->passiveResultPath);
 
         std::ofstream fout;
         fout.open(profilerState->passiveResultPath, std::ios::out|std::ios::binary);
-        fout.write(tmpBytes, static_cast<long>(tmpSize));
+        if (!fout.write(tmpBytes, static_cast<long>(tmpSize)))
+            printf("failure while saving the file\n");
         fout.close();
+
+        delete tmpBytes;
     }
 
 #ifdef _LOGGING
     close_log();
 #endif
 
+    delete profilerState;
+
     if (this->corProfilerInfo != nullptr)
     {
         this->corProfilerInfo->Release();
@@ -214,7 +226,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::FunctionUnloadStarted(FunctionID function
 HRESULT STDMETHODCALLTYPE CorProfiler::JITCompilationStarted(FunctionID functionId, BOOL fIsSafeToBlock)
 {
     // the process was finished, ignoring all firther requests
-    if (profilerState->isFinished) return S_OK;
+    if (std::atomic_load(&shutdownInOrder)) return S_OK;
 
     std::atomic_fetch_add(&shutdownBlockingRequestsCount, 1);
 
@@ -421,6 +433,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::RootReferences(ULONG cRootRefs, ObjectID
 
 HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionThrown(ObjectID thrownObjectId)
 {
+    if (std::atomic_load(&shutdownInOrder)) return S_OK;
     auto exceptionName = GetObjectTypeName(thrownObjectId);
     LOG(
         if(profilerState->threadTracker->hasMapping()) {
@@ -454,7 +467,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionSearchFunctionLeave()
 
 HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionSearchFilterEnter(FunctionID functionId)
 {
-    if (profilerState->isFinished || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
+    if (std::atomic_load(&shutdownInOrder) || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
     LOG(tout << "EXCEPTION Search filter enter");
     profilerState->threadTracker->filterEnter();
     UNUSED(functionId);
@@ -463,7 +476,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionSearchFilterEnter(FunctionID fun
 
 HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionSearchFilterLeave()
 {
-    if (profilerState->isFinished || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
+    if (std::atomic_load(&shutdownInOrder) || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
     LOG(tout << "EXCEPTION Search filter leave");
     profilerState->threadTracker->filterLeave();
     return S_OK;
@@ -502,7 +515,8 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionUnwindFunctionLeave()
 {
     LOG(tout << "EXCEPTION UNWIND FUNCTION LEAVE");
     // the process was finished, ignoring all further requests
-    if (profilerState->isFinished || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
+    if (std::atomic_load(&shutdownInOrder)
+        || !profilerState->threadTracker->isCurrentThreadTracked()) return S_OK;
     profilerState->threadTracker->unwindFunctionLeave();
     return S_OK;
 }
@@ -719,6 +733,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::DynamicMethodJITCompilationFinished(Funct
 }
 
 std::string CorProfiler::GetObjectTypeName(ObjectID objectId) {
+    if (std::atomic_load(&shutdownInOrder)) return "";
     ClassID classId;
     corProfilerInfo->GetClassFromObject(objectId, &classId);
 
@@ -739,6 +754,7 @@ std::string CorProfiler::GetObjectTypeName(ObjectID objectId) {
 }
 
 std::string CorProfiler::GetFunctionName(FunctionID functionId) {
+    if (std::atomic_load(&shutdownInOrder)) return "";
     ClassID classId;
     ModuleID moduleId;
     mdToken token;

VSharp.CoverageInstrumenter/profiler/corProfiler.cpp

@@ -13,6 +13,11 @@ void MethodInfo::serialize(std::vector<char>& buffer) const {
     serializePrimitive(moduleNameLength, buffer);
     serializePrimitiveArray(moduleName, moduleNameLength, buffer);
 }
+
+void MethodInfo::Dispose() {
+    delete moduleName;
+    delete assemblyName;
+}
 //endregion
 
 //region CoverageRecord
@@ -21,27 +26,31 @@ void CoverageRecord::serialize(std::vector<char>& buffer) const {
     serializePrimitive(event, buffer);
     serializePrimitive(methodId, buffer);
     serializePrimitive(thread, buffer);
+    serializePrimitive(timestamp, buffer);
 }
 //endregion
 
+long long GetMicrosecondTime() {
+    using namespace std::chrono;
+    return duration_cast<microseconds>(system_clock::now().time_since_epoch()).count();
+}
+
 //region CoverageHistory
 CoverageHistory::CoverageHistory(OFFSET offset, int methodId) {
     auto insertResult = visitedMethods.insert(methodId);
-    LOG(if (insertResult.second) {
-        tout << "Visit method: " << methodId;
-    });
-    auto record = new CoverageRecord({offset, EnterMain, profilerState->threadInfo->getCurrentThread(), methodId});
+    if (insertResult.second) {
+        LOG(tout << "Visit method: " << methodId);
+    }
+    auto record = new CoverageRecord({offset, EnterMain, profilerState->threadInfo->getCurrentThread(), methodId, GetMicrosecondTime()});
     records.push_back(record);
 }
 
 void CoverageHistory::addCoverage(OFFSET offset, CoverageEvent event, int methodId) {
     auto insertResult = visitedMethods.insert(methodId);
-    LOG(
     if (insertResult.second) {
-        tout << "Visit method: " << methodId;
+        LOG(tout << "Visit method: " << methodId);
     }
-    );
-    auto record = new CoverageRecord({offset, event, profilerState->threadInfo->getCurrentThread(), methodId});
+    auto record = new CoverageRecord({offset, event, profilerState->threadInfo->getCurrentThread(), methodId, GetMicrosecondTime()});
     records.push_back(record);
 }
 
@@ -54,6 +63,8 @@ void CoverageHistory::serialize(std::vector<char>& buffer) const {
 }
 
 CoverageHistory::~CoverageHistory() {
+    for (auto r : records)
+        delete r;
     records.clear();
 }
 //endregion
@@ -99,6 +110,7 @@ void CoverageTracker::invocationFinished() {
         coverage->serialize(buffer);
     }
 
+    delete coverage;
     trackedCoverage->remove();
 
     serializedCoverageMutex.lock();
@@ -110,6 +122,7 @@ void CoverageTracker::invocationFinished() {
 char* CoverageTracker::serializeCoverageReport(size_t* size) {
     collectedMethodsMutex.lock();
     serializedCoverageMutex.lock();
+    printf("got locks, converting info\n");
 
     auto buffer = std::vector<char>();
     auto methodsToSerialize = std::vector<std::pair<int, MethodInfo>>();
@@ -126,6 +139,7 @@ char* CoverageTracker::serializeCoverageReport(size_t* size) {
         serializePrimitive(el.first, buffer);
         el.second.serialize(buffer);
     }
+    printf("converted methods: %lld\n", buffer.size());
 
     auto threadMapping = profilerState->threadTracker->getMapping();
     for (auto mapping: threadMapping) {
@@ -148,16 +162,25 @@ char* CoverageTracker::serializeCoverageReport(size_t* size) {
         serializePrimitiveArray(&serializedCoverage[i][0], serializedCoverage[i].size(), buffer);
     }
 
+    printf("converted reports: %lld\n", buffer.size());
+
+    for (auto cov : trackedCoverage->items())
+        delete cov.second;
     trackedCoverage->clear();
     serializedCoverage.clear();
     methodsToSerialize.clear();
 
     serializedCoverageMutex.unlock();
     collectedMethodsMutex.unlock();
 
+    printf("cleared and unlocked data\n");
+
     *size = buffer.size();
     char* array = new char[*size];
+    printf("successfully allocated\n");
     std::memcpy(array, &buffer[0], *size);
+
+    printf("total bytes: %lld", *size);
     return array;
 }
 
@@ -179,9 +202,15 @@ void CoverageTracker::clear()  {
 
 CoverageTracker::~CoverageTracker(){
     clear();
+    for (int i = 0; i < collectedMethods.size(); i++)
+        collectedMethods[i].Dispose();
+    collectedMethods.clear();
+    delete trackedCoverage;
 }
 
 void CoverageTracker::invocationAborted() {
+    auto abortedCov = trackedCoverage->load();
+    delete abortedCov;
     trackedCoverage->update([](CoverageHistory *cov) {
         return (CoverageHistory*) nullptr;
     });

VSharp.CoverageInstrumenter/profiler/coverageTracker.cpp

@@ -19,7 +19,8 @@ enum CoverageEvent {
     Call,
     Tailcall,
     TrackCoverage,
-    StsfldHit
+    StsfldHit,
+    ThrowLeave,
 };
 
 struct MethodInfo {
@@ -31,13 +32,17 @@ struct MethodInfo {
     std::string methodName;
 
     void serialize(std::vector<char>& buffer) const;
+
+    // frees previously allocated resources for it; the object is not supposed to be used afterwards
+    void Dispose();
 };
 
 struct CoverageRecord {
     OFFSET offset;
     CoverageEvent event;
     ThreadID thread;
     int methodId;
+    long long timestamp;
 
     void serialize(std::vector<char>& buffer) const;
 };
@@ -63,9 +68,9 @@ class CoverageTracker {
     ThreadStorage<CoverageHistory*>* trackedCoverage;
     ThreadTracker* threadTracker;
     std::mutex serializedCoverageMutex;
-    std::vector<std::vector<char>> serializedCoverage;
     std::vector<int> serializedCoverageThreadIds;
 public:
+    std::vector<std::vector<char>> serializedCoverage;
     std::mutex collectedMethodsMutex;
     std::vector<MethodInfo> collectedMethods;
     explicit CoverageTracker(ThreadTracker* threadTracker, ThreadInfo* threadInfo, bool collectMainOnly);