Cutlass CUDA Templates for Linear Algebra Subroutines and Solvers

fragment_stream.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include <cutlass/convert.h>
32 #include <cutlass/fragment.h>
33 #include <cutlass/iterator_access.h>
34 #include <cutlass/tensor_ref.h>
35 #include <cutlass/tile_iterator.h>
36
37 #include <cutlass/gemm/gemm_operand.h>
38 #include <cutlass/gemm/gemm_traits.h>
39 #include <cutlass/matrix_traits.h>
40
41 namespace cutlass {
42
44
46 template <typename Traits_, typename LoadIterator_, typename StoreIterator_,
47 typename Convert_ =
48 FragmentCopy<typename StoreIterator_::Fragment, typename LoadIterator_::Fragment>,
49 typename Index_ = int>
50 struct FragmentStream {
51 //
52 // Constant and type definitions
53 //
54
56 typedef Traits_ Traits;
57
59 typedef LoadIterator_ LoadIterator;
60
62 typedef StoreIterator_ StoreIterator;
63
65 typedef Convert_ Convert;
66
68 typedef Index_ Index;
69
70 //
71 // Dependent types
72 //
73
75 typedef typename LoadIterator::Fragment Fragment;
76
78 typedef typename StoreIterator::Fragment StoreFragment;
79
81 typedef typename StoreIterator::Storage Storage;
82
84 typedef typename StoreIterator::Storage SharedStoreStorage;
85
86 //
87 // Nested classes
88 //
89
91 struct Params {
93 typedef typename LoadIterator::Params LoadParams;
94
96 typedef typename StoreIterator::Params StoreParams;
97
98 //
99 // Data members
100 //
101
103 LoadParams load_params;
104
106 StoreParams store_params;
107
108 //
109 // Methods
110 //
111
113 CUTLASS_HOST_DEVICE int initialize(LoadParams const &_load_params,
114 StoreParams const &_store_params) {
115 load_params = _load_params;
116 store_params = _store_params;
117 return 0;
118 }
119 };
120
121 //
122 // Data members
123 //
124
126 LoadIterator load_iterator;
127
129 typename LoadIterator::PredicateVector predicates;
130
132 StoreIterator store_iterator;
133
135 Fragment fetch;
136
138 Convert convert;
139
140 //
141 // Static members
142 //
143
145 static CUTLASS_DEVICE void shared_store_fence() { __syncthreads(); }
146
147 //
148 // Methods
149 //
150
151 CUTLASS_DEVICE
152 FragmentStream() {}
153
155 CUTLASS_DEVICE
156 FragmentStream(Params const &params, Coord<3> const &bounds,
157 Coord<3> const &block_offset = make_Coord(0, 0, 0))
158 : load_iterator(params.load_params, block_offset), store_iterator(params.store_params) {
159 // set predicates
160 initialize_predicates(bounds, block_offset);
161
162 fetch.clear();
163 }
164
166 CUTLASS_DEVICE
167 void load() {
168 ConstPredicateTileAdapter<typename LoadIterator::PredicateVector,
169 typename LoadIterator::Iterations>
170 predicates_it(predicates);
171 iterator_load_post_increment(load_iterator, fetch, predicates_it);
172 }
173
175 CUTLASS_DEVICE
176 void commit() {
177 StoreFragment store_fragment(convert(fetch));
178
179 iterator_store(store_iterator, store_fragment);
180 }
181
183 CUTLASS_DEVICE
184 void initialize_predicates(Coord<3> const &bounds, Coord<3> const &block_offset) {
185 PredicateTileAdapter<typename LoadIterator::PredicateVector, typename LoadIterator::Iterations>
186 predicates_it(predicates);
187 load_iterator.initialize_predicates(predicates_it, bounds, block_offset);
188 }
189 };
190
192
193 } // namespace cutlass
cutlass::FragmentStream::store_iterator
StoreIterator store_iterator
Stores fragment to shared memory.
Definition: fragment_stream.h:132
cutlass::TileStoreIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kShared, Traits_::Index, Traits_::Scalar, IteratorFragment::kScalar, Traits_::DestinationSkew >::Fragment
Base::Fragment Fragment
Fragment definition.
Definition: tile_iterator.h:606
cutlass
Definition: convert.h:34
tensor_ref.h
Defines a structure containing strides, bounds, and a pointer to tensor data.
cutlass::FragmentStream::Params::store_params
StoreParams store_params
Parameters to the store iterator.
Definition: fragment_stream.h:106
cutlass::FragmentStream::predicates
LoadIterator::PredicateVector predicates
Predicate vector.
Definition: fragment_stream.h:129
tile_iterator.h
Defines the Tile Traits concept and iterators for loading and storing to tiles efficiently.
gemm_traits.h
Defines structural properties of complete GEMM computation.
cutlass::make_Coord
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:241
cutlass::FragmentStream::Storage
StoreIterator::Storage Storage
Destination storage.
Definition: fragment_stream.h:81
cutlass::FragmentStream::FragmentStream
CUTLASS_DEVICE FragmentStream(Params const &params, Coord< 3 > const &bounds, Coord< 3 > const &block_offset=make_Coord(0, 0, 0))
Constructor.
Definition: fragment_stream.h:156
cutlass::FragmentStream::fetch
Fragment fetch
Fragment fetched by load iterator.
Definition: fragment_stream.h:135
cutlass::FragmentStream::commit
CUTLASS_DEVICE void commit()
Commits the fragment.
Definition: fragment_stream.h:176
cutlass::FragmentStream::StoreIterator
StoreIterator_ StoreIterator
Defines the store iterator.
Definition: fragment_stream.h:62
cutlass::PredicateTileAdapter
Adapter to enable random access to predicates via logical coordinate within a tile.
Definition: predicate_vector.h:435
cutlass::FragmentStream::Index
Index_ Index
Index type.
Definition: fragment_stream.h:68
cutlass::TileIteratorBase< Traits_, Traits_::Scalar, Advance_, MemorySpace, Traits_::Index, Traits_::Scalar, IteratorFragment_, Traits_::DestinationSkew >::Storage
Fragment< Scalar, ShapeCount< Tile >::kCount, kFragmentSize > Storage
The storage.
Definition: tile_iterator.h:181
cutlass::TileLoadIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kGlobal, Traits_::Index >::Iterations
Base::Iterations Iterations
Iterations.
Definition: tile_iterator.h:346
cutlass::FragmentStream::shared_store_fence
static CUTLASS_DEVICE void shared_store_fence()
The memory fence for shared stores.
Definition: fragment_stream.h:145
iterator_access.h
Free functions for loading and storing to implementations of tile iteartor concepts.
cutlass::TileLoadIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kGlobal, Traits_::Index >
cutlass::iterator_load_post_increment
CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, Fragment &fragment, typename InputIterator::Index offset, ConstPredicateAdapter predicate_adapter)
Loads a fragment from an input iterator, masked by a predicate iterator.
Definition: iterator_access.h:113
cutlass::FragmentStream::SharedStoreStorage
StoreIterator::Storage SharedStoreStorage
The storage in shared memory.
Definition: fragment_stream.h:84
cutlass::FragmentStream::initialize_predicates
CUTLASS_DEVICE void initialize_predicates(Coord< 3 > const &bounds, Coord< 3 > const &block_offset)
Recomputes predicates.
Definition: fragment_stream.h:184
gemm_operand.h
Defines constant expressions for mapping GEMM problem size and strides onto pitch-linear memory...
cutlass::FragmentStream
Manages a pair of iterators to stream data from global memory to shared.
Definition: fragment_stream.h:50
cutlass::FragmentStream::FragmentStream
CUTLASS_DEVICE FragmentStream()
Definition: fragment_stream.h:152
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:37
cutlass::Coord
Statically-sized array specifying Coords within a tensor.
Definition: coord.h:48
cutlass::iterator_store
CUTLASS_HOST_DEVICE void iterator_store(OutputIterator &iterator, Fragment &fragment)
Stores a fragment to an output iterator.
Definition: iterator_access.h:174
cutlass::FragmentStream::Params::LoadParams
LoadIterator::Params LoadParams
Load parameters.
Definition: fragment_stream.h:93
cutlass::FragmentStream::Convert
Convert_ Convert
Converts between tiles.
Definition: fragment_stream.h:65
cutlass::FragmentStream::StoreFragment
StoreIterator::Fragment StoreFragment
Stored fragment type.
Definition: fragment_stream.h:78
cutlass::FragmentStream::Traits
Traits_ Traits
Defines traits of WMMA GEMM tile stream.
Definition: fragment_stream.h:56
cutlass::FragmentStream::load
CUTLASS_DEVICE void load()
Loads the fragment.
Definition: fragment_stream.h:167
cutlass::FragmentStream::Params::StoreParams
StoreIterator::Params StoreParams
Store parameters.
Definition: fragment_stream.h:96
cutlass::ConstPredicateTileAdapter
Adapter to enable random access to predicates via logical coordinate within a tile.
Definition: predicate_vector.h:466
cutlass::FragmentStream::Fragment
LoadIterator::Fragment Fragment
Loaded fragment type.
Definition: fragment_stream.h:75
cutlass::FragmentStream::LoadIterator
LoadIterator_ LoadIterator
Defines the load iterator.
Definition: fragment_stream.h:59
cutlass::FragmentStream::Params::initialize
CUTLASS_HOST_DEVICE int initialize(LoadParams const &_load_params, StoreParams const &_store_params)
Initializes parameters.
Definition: fragment_stream.h:113
matrix_traits.h
Defines properties of matrices used to denote layout and operands to GEMM kernels.
cutlass::FragmentStream::Params::load_params
LoadParams load_params
Parameters to load iterator.
Definition: fragment_stream.h:103
cutlass::FragmentStream::convert
Convert convert
Converts between load fragments and store fragments.
Definition: fragment_stream.h:138
fragment.h
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...
cutlass::FragmentStream::load_iterator
LoadIterator load_iterator
Loads fragment from global memory.
Definition: fragment_stream.h:126
convert.h
Defines conversion operations among Fragments of different base type.
cutlass::TileStoreIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kShared, Traits_::Index, Traits_::Scalar, IteratorFragment::kScalar, Traits_::DestinationSkew >
cutlass::FragmentStream::Params
Parameters passed to initialize the ierator.
Definition: fragment_stream.h:91