Debug 8M element matrix

packages/ifpack2/src/Ifpack2_IlukGraph.hpp

@@ -541,12 +541,22 @@ void IlukGraph<GraphType, KKHandleType>::initialize(const Teuchos::RCP<KKHandleT
   const int NumMyRows    = OverlapGraph_->getRowMap()->getLocalNumElements();
   auto localOverlapGraph = OverlapGraph_->getLocalGraphDevice();
 
+  // VINH TEST
+  //int myRank;
+  //MPI_Comm_rank (MPI_COMM_WORLD, &myRank);
+  // END VINH TEST
+
   if (KernelHandle->get_spiluk_handle()->get_nrows() < static_cast<size_type>(NumMyRows)) {
     KernelHandle->get_spiluk_handle()->reset_handle(NumMyRows,
                                                     KernelHandle->get_spiluk_handle()->get_nnzL(),
                                                     KernelHandle->get_spiluk_handle()->get_nnzU());
   }
 
+  // VINH TEST
+  //fprintf(stderr, "IlukGraph<GraphType, KKHandleType>::initialize () BeFore symbolic, rank %d, NumMyRows %d, num L entries %d, num U entries %d, Overalloc_ %lf\n",
+  //        myRank, NumMyRows, KernelHandle->get_spiluk_handle()->get_nnzL(), KernelHandle->get_spiluk_handle()->get_nnzU(), Overalloc_);
+  // END VINH TEST
+
   lno_row_view_t L_row_map("L_row_map", NumMyRows + 1);
   lno_nonzero_view_t L_entries("L_entries", KernelHandle->get_spiluk_handle()->get_nnzL());
   lno_row_view_t U_row_map("U_row_map", NumMyRows + 1);
@@ -564,6 +574,10 @@ void IlukGraph<GraphType, KKHandleType>::initialize(const Teuchos::RCP<KKHandleT
       data_type nnzL = static_cast<data_type>(Overalloc_) * L_entries.extent(0);
       data_type nnzU = static_cast<data_type>(Overalloc_) * U_entries.extent(0);
       KernelHandle->get_spiluk_handle()->reset_handle(NumMyRows, nnzL, nnzU);
+      // VINH TEST
+      //fprintf(stderr, "IlukGraph<GraphType, KKHandleType>::initialize () symbolic caught error, rank %d, NumMyRows %d, num L entries %d, num U entries %d, Overalloc_ %lf\n",
+      //        myRank, NumMyRows, KernelHandle->get_spiluk_handle()->get_nnzL(), KernelHandle->get_spiluk_handle()->get_nnzU(), Overalloc_);
+      // END VINH TEST
       Kokkos::resize(L_entries, KernelHandle->get_spiluk_handle()->get_nnzL());
       Kokkos::resize(U_entries, KernelHandle->get_spiluk_handle()->get_nnzU());
     }
@@ -574,6 +588,11 @@ void IlukGraph<GraphType, KKHandleType>::initialize(const Teuchos::RCP<KKHandleT
     symbolicError = globalSymbolicError > 0;
   } while (symbolicError);
 
+  // VINH TEST
+  //fprintf(stderr, "IlukGraph<GraphType, KKHandleType>::initialize () AfTer symbolic, rank %d, NumMyRows %d, num L entries %d, num U entries %d\n",
+  //        myRank, NumMyRows, KernelHandle->get_spiluk_handle()->get_nnzL(), KernelHandle->get_spiluk_handle()->get_nnzU());
+  // END VINH TEST
+
   Kokkos::resize(L_entries, KernelHandle->get_spiluk_handle()->get_nnzL());
   Kokkos::resize(U_entries, KernelHandle->get_spiluk_handle()->get_nnzU());
 

packages/ifpack2/src/Ifpack2_RILUK_def.hpp

@@ -492,6 +492,11 @@ void RILUK<MatrixType>::initialize() {
                                                                                   "Tpetra::CrsMatrix, please call fillComplete on it (with the domain and "
                                                                                   "range Maps, if appropriate) before calling this method.");
 
+  // VINH TEST
+  int myRank, nRanks;
+  MPI_Comm_rank (MPI_COMM_WORLD, &myRank);
+  MPI_Comm_size (MPI_COMM_WORLD, &nRanks);
+  // END VINH TEST
   Teuchos::Time timer("RILUK::initialize");
   double startTime = timer.wallTime();
   {  // Start timing
@@ -521,6 +526,12 @@ void RILUK<MatrixType>::initialize() {
     }
 
     A_local_ = makeLocalFilter(A_);
+    // VINH TEST
+    fprintf(stderr, "RILUK<MatrixType>::initialize (), rank %d, A_ global (%d x %d, %d), local (%d x %d, %d), A_local_ global (%d x %d, %d), local (%d x %d, %d)\n",
+            myRank, A_->getGlobalNumRows(), A_->getGlobalNumCols(), A_->getGlobalNumEntries(), A_->getLocalNumRows(), A_->getLocalNumCols(), A_->getLocalNumEntries(),
+            A_local_->getGlobalNumRows(), A_local_->getGlobalNumCols(), A_local_->getGlobalNumEntries(),
+            A_local_->getLocalNumRows(), A_local_->getLocalNumCols(), A_local_->getLocalNumEntries());
+    // END VINH TEST
 
     TEUCHOS_TEST_FOR_EXCEPTION(
         A_local_.is_null(), std::logic_error,
@@ -558,6 +569,11 @@ void RILUK<MatrixType>::initialize() {
         A_local_crs_ = rcp_const_cast<const crs_matrix_type>(A_local_crs_nc_);
       }
       if (!isKokkosKernelsStream_) {
+        {// VINH TEST
+          //auto lclMtx = A_local_crs_->getLocalMatrixDevice();      
+          //std::string fnA_mtx = "A_r_" + std::to_string( myRank ) + ".mtx";
+          //KokkosSparse::Impl::write_kokkos_crst_matrix<local_matrix_device_type>(lclMtx, fnA_mtx.c_str());
+        }// VINH TEST
         Graph_ = rcp(new Ifpack2::IlukGraph<crs_graph_type, kk_handle_type>(A_local_crs_->getCrsGraph(),
                                                                             LevelOfFill_, 0, Overalloc_));
       } else {
@@ -639,6 +655,9 @@ void RILUK<MatrixType>::initialize() {
       }
     }
 
+    // VINH TEST
+    //fprintf(stderr, "RILUK<MatrixType>::initialize (), rank %d, BEFORE create_spiluk_handle and spiluk_symbolic, LevelOfFill_ %d\n", myRank, LevelOfFill_);
+    // END VINH TEST
     if (this->isKokkosKernelsSpiluk_) {
       if (!isKokkosKernelsStream_) {
         this->KernelHandle_ = Teuchos::rcp(new kk_handle_type());
@@ -661,6 +680,9 @@ void RILUK<MatrixType>::initialize() {
     } else {
       Graph_->initialize();
     }
+    // VINH TEST
+    //fprintf(stderr, "RILUK<MatrixType>::initialize (), rank %d, AFTER create_spiluk_handle and spiluk_symbolic\n", myRank);
+    // END VINH TEST
 
     allocate_L_and_U();
     checkOrderingConsistency(*A_local_);