Synchronize access to cachedVBStates between a flusher task commit and a snapshot task

SUMMARY: ThreadSanitizer: data race /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/kvstore.cc:289 vbucket_state::toJSON() const
 ====================================
WARNING: ThreadSanitizer: data race (pid=209164)
Read of size 8 at 0x7d1400009e50 by thread T16 (mutexes: write M16145):

Previous write of size 8 at 0x7d1400009e50 by thread T18 (mutexes: write M16141):

Change-Id: Ie5c41c4e4ba0219903b85047787ece373c6a8231
Reviewed-on: http://review.couchbase.org/65988
Reviewed-by: Dave Rigby <[email protected]>
Tested-by: Sriram Ganesan <[email protected]>

Author: Sriram Ganesan

src/forest-kvstore/forest-kvstore.cc

@@ -1408,6 +1408,8 @@ bool ForestKVStore::snapshotVBucket(uint16_t vbucketId, vbucket_state& vbstate,
 
     hrtime_t start = gethrtime();
 
+    LockHolder lh(writerLock);
+
     if (updateCachedVBState(vbucketId, vbstate) &&
          (options == VBStatePersist::VBSTATE_PERSIST_WITHOUT_COMMIT ||
           options == VBStatePersist::VBSTATE_PERSIST_WITH_COMMIT)) {
@@ -1423,7 +1425,6 @@ bool ForestKVStore::snapshotVBucket(uint16_t vbucketId, vbucket_state& vbstate,
         statDoc.body = const_cast<char *>(stateStr.c_str());
         statDoc.bodylen = stateStr.length();
 
-        LockHolder lh(writerLock);
         fdb_status status = fdb_set(writeVbStateHandle, &statDoc);
 
         if (status != FDB_RESULT_SUCCESS) {