Timers in Dw

src/Dirac/Dirac.jl

@@ -56,17 +56,24 @@ export DiracWorkspace, DiracParam
 function Dw!(so, U, si, dpar::DiracParam, lp::SpaceParm{4,6,B,D}, ymws = "ymws", gp = "gp") where {B,D}
 
     if B == BC_SF_AFWB || B == BC_SF_ORBI
+
+        @timeit "SF fix" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(si,si,1.0,Gamma{16},lp)
         end
+        end
+        
         @timeit "Dw" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_Dw!(so, U, si, dpar.m0, dpar.th, lp)
             end
         end
+
+        @timeit "SF fix" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(so,si,dpar.ct,Gamma{16},lp)
         end 
+        end
 
     else
         @timeit "Dw" begin
@@ -76,11 +83,11 @@ function Dw!(so, U, si, dpar::DiracParam, lp::SpaceParm{4,6,B,D}, ymws = "ymws",
             end
     end 
 
+    @timeit "Dw_improvement" begin
     if abs(dpar.csw) > 1.0E-10
         if ymws == "ymws" || gp == "gp"
             error("YM workspace and Gauge parameters must be included for the improved dirac operator.")
         else
-        @timeit "Dw_improvement" begin
             for i in 1:Int(lp.npls/2)
                 z1 = exp(2im * pi * lp.ntw[i]/gp.ng)
                 z2 = exp(2im * pi * lp.ntw[lp.npls-i]/gp.ng)
@@ -101,19 +108,27 @@ function DwdagDw!(so, U, si, dpar::DiracParam, st, lp::SpaceParm{4,6,B,D}, ymws
     
 
     if B == BC_SF_AFWB || B == BC_SF_ORBI
-        CUDA.@sync begin
-            CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(si,si,1.0,Gamma{16},lp)
-        end
         @timeit "DwdagDw" begin
+        
+            @timeit "SF fix" begin
+            CUDA.@sync begin
+                CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(si,si,1.0,Gamma{16},lp)
+            end
+            end
+
+            @timeit "g5Dw" begin
             CUDA.@sync begin
                 CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_g5Dw!(st, U, si, dpar.m0, dpar.th, lp)
             end
+            end
 
+            @timeit "SF fix" begin
             CUDA.@sync begin
                 CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(st,si,dpar.ct,Gamma{5},lp)
             end
+            end
 
-
+            @timeit "Dw_mprovement" begin
             if abs(dpar.csw) > 1.0E-10
                 if ymws == "ymws" || gp == "gp"
                     error("YM workspace and Gauge parameters must be included for the improved dirac operator.")
@@ -130,21 +145,27 @@ function DwdagDw!(so, U, si, dpar::DiracParam, st, lp::SpaceParm{4,6,B,D}, ymws
                 end
                 end
             end
+            end
 
 
+            @timeit "g5Dw" begin
             CUDA.@sync begin
                 CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_g5Dw!(so, U, st, dpar.m0, dpar.th, lp)
             end
+            end
 
+
+            @timeit "SF fix" begin
             CUDA.@sync begin
                 CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(so,st,dpar.ct,Gamma{5},lp)
             end
+            end
 
+            @timeit "Dw_improvement" begin
             if abs(dpar.csw) > 1.0E-10
                 if ymws == "ymws" || gp == "gp"
                     error("YM workspace and Gauge parameters must be included for the improved dirac operator.")
                 else
-                @timeit "Dw_improvement" begin
                     for i in 1:Int(lp.npls/2)
                         z1 = exp(2im * pi * lp.ntw[i]/gp.ng)
                         z2 = exp(2im * pi * lp.ntw[lp.npls-i]/gp.ng)
@@ -158,7 +179,7 @@ function DwdagDw!(so, U, si, dpar::DiracParam, st, lp::SpaceParm{4,6,B,D}, ymws
             end                
 
             
-            end
+        end
 
     else
         @timeit "DwdagDw" begin
@@ -178,20 +199,23 @@ end
 function g5Dw!(so, U, si, dpar, lp::SpaceParm{4,6,B,D}, ymws = "ymws", gp = "gp") where {B,D}
 
     if B == BC_SF_AFWB || B == BC_SF_ORBI
+        @timeit "SF fix" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(si,si,1.0,Gamma{16},lp)
         end
-        @timeit "Dw" begin
+        end
+        @timeit "g5Dw" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_g5Dw!(so, U, si, dpar.m0, dpar.th , lp)
             end
         end
+        @timeit "SF fix" begin
         CUDA.@sync begin
             CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_sfbndfix!(so,si,dpar.ct,Gamma{5},lp)
         end 
-        
+        end
     else
-        @timeit "Dw" begin
+        @timeit "g5Dw" begin
             CUDA.@sync begin
                 CUDA.@cuda threads=lp.bsz blocks=lp.rsz krnl_g5Dw!(so, U, si, dpar.m0, dpar.th , lp)
                 end
@@ -199,11 +223,11 @@ function g5Dw!(so, U, si, dpar, lp::SpaceParm{4,6,B,D}, ymws = "ymws", gp = "gp"
     end 
 
 
+    @timeit "Dw_improvement" begin
     if abs(dpar.csw) > 1.0E-10
         if ymws == "ymws" || gp == "gp"
             error("YM workspace and Gauge parameters must be included for the improved dirac operator.")
         else
-        @timeit "Dw_improvement" begin
             for i in 1:Int(lp.npls/2)
                 z1 = exp(2im * pi * lp.ntw[i]/gp.ng)
                 z2 = exp(2im * pi * lp.ntw[lp.npls-i]/gp.ng)