Cutlass CUDA Templates for Linear Algebra Subroutines and Solvers

batched_reduction.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #if !defined(__CUDACC_RTC__)
32 #include <cuda.h>
33 #endif
34
35 #include "cutlass/coord.h"
36 #include "cutlass/util/platform.h"
37 #include "cutlass/fragment.h"
38
39 namespace cutlass {
40 namespace reduction {
41
43
44 template <typename batched_reduction_>
45 __global__ __launch_bounds__(batched_reduction_::Traits::kThreads, 1) void batched_reduction_kernel(typename batched_reduction_::Params params) {
46 // Construct the batched_reduction object
47 batched_reduction_ batched_reduction(params);
48 batched_reduction.run();
49 }
50
51 template <typename BatchedReductionTraits_>
52 struct BatchedReduction {
54 typedef BatchedReduction<BatchedReductionTraits_> This_;
56 typedef BatchedReductionTraits_ Traits;
58 typedef typename Traits::Params Params;
60 typedef typename Traits::Functor Functor;
61
63 CUTLASS_DEVICE BatchedReduction(Params const &params_)
64 : params(params_), functor(params_.functorParams) {}
65
68 CUTLASS_DEVICE void run() {
69 #if (__CUDA_ARCH__ >= 600)
70 // Swizzle the IDs of the block
71 typename Traits::BlockSwizzle block_swizzle;
72 Coord<3> threadblock_offset =
73 block_swizzle.get_threadblock_offset(make_Coord_from_shape<Traits::SubTile>());
74
75 int subTileSize = gridDim.x * Traits::SubTile::kW;
76 int tileSize = params.problem_size[1] * params.problem_size[2];
77 int subTileOffset = threadblock_offset[2] + threadIdx.x * Traits::ThreadShape::kW;
78
79 int subTileBase = 0;
80
81 typename Traits::ScalarA inRegs[Traits::maxInReg];
82 typename Traits::ScalarAccum AccumRegs[Traits::maxOutReg];
83
84 for (int subTile = 0; subTile < tileSize; subTile += subTileSize) {
85 int tileOffset = subTileBase + subTileOffset;
86 // Init AccumRegs
87 for (int i = 0; i < Traits::ThreadShape::kW; i++)
88 AccumRegs[i] = static_cast<typename Traits::ScalarAccum>(0.0f);
89 // Fetch c0
90 typename Traits::ScalarAccum c0[Traits::ThreadShape::kW];
91 for (int i = 0; i< Traits::ThreadShape::kW; i++)
92 c0[i] = static_cast<typename Traits::ScalarAccum>(params.d_c[tileOffset + i]);
93
94 // Fetch partial sums from A
95 #pragma unroll
96 for (int s = 0; s < Traits::ReductionSize; s++) {
97 int inRegOffset = s * Traits::ThreadShape::kW;
98 int dOffset = (s * tileSize) + tileOffset;
99 #pragma unroll
100 for (int i = 0; i< Traits::ThreadShape::kW; i++) {
101 inRegs[inRegOffset + i] = params.d_a[dOffset + i];
102 }
103 }
104
105 // Accumulate
106 #pragma unroll
107 for (int s = 0; s < Traits::ReductionSize; s++) {
108 int inRegOffset = s * Traits::ThreadShape::kW;
109 #pragma unroll
110 for (int i = 0; i < Traits::ThreadShape::kW; i++) {
111 //AccumRegs[i] = cuFma(params.alpha, inRegs[inRegOffset + i], AccumRegs[i]);
112 //AccumRegs[i] = params.alpha * inRegs[inRegOffset + i] + AccumRegs[i];
113 AccumRegs[i] = static_cast<typename Traits::ScalarAccum>(inRegs[inRegOffset + i]) + AccumRegs[i];
114 }
115 }
116 // calling functor
117 functor_caller<Traits::ThreadShapeMultiple2>(AccumRegs, c0, AccumRegs);
118
119 // Store AccumRegs to D
120 #pragma unroll
121 for (int i = 0; i < Traits::ThreadShape::kW; i++) {
122 params.d_d[tileOffset + i] = static_cast<typename Traits::ScalarD>(AccumRegs[i]);
123 }
124
125 // Advance sub-tile pointer
126 subTileBase += subTileSize;
127 } // end for loop
128 #endif //#if (__CUDA_ARCH__ >= 600)
129 }
130
131 template<bool ThreadShapeMultiple2>
132 CUTLASS_DEVICE void functor_caller(typename Traits::ScalarAccum const *accum, typename Traits::ScalarAccum const *old, typename Traits::ScalarAccum *output) {
133 if (ThreadShapeMultiple2 == true) {
134 for (int i = 0; i < Traits::ThreadShape::kW / 2; i++) {
135 functor.template evaluate<typename Traits::ScalarAccum, typename Traits::ScalarAccum, 2>(&accum[2 * i], &old[2 * i], &output[2 * i]);
136 }
137 }
138 else {
139 for (int i = 0; i < Traits::ThreadShape::kW; i++) {
140 functor.template evaluate<typename Traits::ScalarAccum, typename Traits::ScalarAccum, 1>(&accum[i], &old[i], &output[i]);
141 }
142 }
143 }
144
145 //
146 // Static function members
147 //
148 #if !defined(__CUDACC_RTC__)
149 static __host__ cudaError_t launch(Params const& params,
151 cudaStream_t stream = cudaStreamDefault) {
152 // Setup the grid.
153 typename Traits::BlockSwizzle block_swizzle;
154 dim3 grid = block_swizzle.get_grid_layout(params.problem_size,
155 make_Coord_from_shape<typename Traits::OutputTile>());
156
157 dim3 block;
158 block.x = Traits::kThreads;
159 batched_reduction_kernel<This_><<<grid, block, 0, stream>>>(params);
160 return cudaGetLastError();
161 }
162 #endif
163
164 //
165 // Data members
166 //
167
169 Params const& params;
170 // The functor.
171 Functor functor;
172 };
173
174 } // namespace reduction
175 } // namespace cutlass
cutlass
Definition: convert.h:33
cutlass::reduction::BatchedReduction::params
Params const & params
The params.
Definition: batched_reduction.h:169
cutlass::reduction::__launch_bounds__
__global__ __launch_bounds__(batched_reduction_::Traits::kThreads, 1) void batched_reduction_kernel(typename batched_reduction_
Definition: batched_reduction.h:45
coord.h
A Coord is a coordinate of arbitrary rank into a tensor or matrix.
cutlass::reduction::BatchedReduction::run
CUTLASS_DEVICE void run()
Definition: batched_reduction.h:68
platform.h
C++ features that may be otherwise unimplemented for CUDA device functions.
cutlass::reduction::BatchedReduction::This_
BatchedReduction< BatchedReductionTraits_ > This_
This class.
Definition: batched_reduction.h:54
cutlass::reduction::BatchedReduction::functor
Functor functor
Definition: batched_reduction.h:171
cutlass::reduction::BatchedReduction
Definition: batched_reduction.h:52
cutlass::Coord< 3 >
cutlass::reduction::BatchedReduction::BatchedReduction
CUTLASS_DEVICE BatchedReduction(Params const &params_)
ctor
Definition: batched_reduction.h:63
cutlass::reduction::BatchedReduction::Params
Traits::Params Params
Params.
Definition: batched_reduction.h:58
cutlass::reduction::BatchedReduction::functor_caller
CUTLASS_DEVICE void functor_caller(typename Traits::ScalarAccum const *accum, typename Traits::ScalarAccum const *old, typename Traits::ScalarAccum *output)
Definition: batched_reduction.h:132
cutlass::reduction::BatchedReduction::Functor
Traits::Functor Functor
functor
Definition: batched_reduction.h:60
cutlass::reduction::BatchedReduction::Traits
BatchedReductionTraits_ Traits
The traits.
Definition: batched_reduction.h:56
fragment.h
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...
cutlass::reduction::BatchedReduction::launch
static __host__ cudaError_t launch(Params const &params, cudaStream_t stream=cudaStreamDefault)
Launch the kernel.
Definition: batched_reduction.h:150