Cutlass CUDA Templates for Linear Algebra Subroutines and Solvers

gemm_shared_stream.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include "cutlass/tensor_ref.h"
32 #include "cutlass/gemm/gemm_shared_tile.h"
33
34 namespace cutlass {
35 namespace gemm {
36
38
39 template <
41 typename Iterator_,
43 typename Transformer_ = Copy<typename Iterator_::Fragment> >
44
45 struct SharedLoadStream {
47 typedef Iterator_ Iterator;
49 typedef Transformer_ Transformer;
50
52 typedef typename Iterator::Fragment FetchedFragment;
54 typedef typename Transformer::OutputFragment TransformedFragment;
56 static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value),
57 "");
59 typedef TransformedFragment Fragment;
61 typedef typename Iterator::Scalar Scalar;
62
64 typedef TensorRef<Scalar, 4> TensorRef;
65
67 struct Params {
69 typename Iterator::Params iterator;
70
72 CUTLASS_HOST_DEVICE int initialize() { return iterator.initialize(); }
73 };
74
76 typedef typename Iterator::Storage SharedStorage;
77
79 CUTLASS_DEVICE SharedLoadStream() {}
80
82 CUTLASS_DEVICE SharedLoadStream(Params const &params, TensorRef const &ref) {
83 this->initialize(params, ref);
84 }
85
87 CUTLASS_DEVICE void initialize(Params const &params, TensorRef const &ref) {
88 // The iterator.
89 iterator = Iterator(params.iterator, ref.data());
90 // The transformer.
91 transformer = Transformer();
92 }
93
95 CUTLASS_DEVICE void copy() {
96 iterator.load_post_increment(fetched[0]);
97 }
98
100 CUTLASS_DEVICE void copy(int step) { iterator.load(fetched[step % 2], step); }
101
103 CUTLASS_DEVICE void commit() { transformer.transform(fetched[0], transformed[0]); }
104
106 CUTLASS_DEVICE void commit(int step) {
107 transformer.transform(fetched[step % 2], transformed[step % 2]);
108 }
109
111 CUTLASS_DEVICE TransformedFragment &fragment(int step = 0) { return transformed[step % 2]; }
112
114 CUTLASS_DEVICE TransformedFragment const &fragment(int step = 0) const {
115 return transformed[step % 2];
116 }
117
119 CUTLASS_DEVICE void inc_stage() { iterator.inc_stage(); }
120
122 Iterator iterator;
124 FetchedFragment fetched[2];
126 Transformer transformer;
128 TransformedFragment transformed[2];
129 };
130
132
133 } // namespace gemm
134 } // namespace cutlass
cutlass::gemm::SharedLoadStream::commit
CUTLASS_DEVICE void commit()
Commit the data.
Definition: gemm_shared_stream.h:103
cutlass
Definition: convert.h:33
tensor_ref.h
Defines a structure containing strides, bounds, and a pointer to tensor data.
cutlass::gemm::SharedLoadStream::fetched
FetchedFragment fetched[2]
Fetched fragment.
Definition: gemm_shared_stream.h:124
cutlass::platform::is_same
std::is_same (false specialization)
Definition: platform.h:420
cutlass::gemm::SharedLoadStream::commit
CUTLASS_DEVICE void commit(int step)
Commit the data.
Definition: gemm_shared_stream.h:106
cutlass::gemm::SharedLoadStream::SharedStorage
Iterator::Storage SharedStorage
The storage in shared memory needed by that stream.
Definition: gemm_shared_stream.h:76
cutlass::gemm::SharedLoadStream::inc_stage
CUTLASS_DEVICE void inc_stage()
Increment the stage.
Definition: gemm_shared_stream.h:119
cutlass::gemm::SharedLoadStream::TensorRef
TensorRef< Scalar, 4 > TensorRef
Reference type to a tensor.
Definition: gemm_shared_stream.h:64
cutlass::gemm::SharedLoadStream::SharedLoadStream
CUTLASS_DEVICE SharedLoadStream()
Ctor.
Definition: gemm_shared_stream.h:79
cutlass::gemm::SharedLoadStream::transformed
TransformedFragment transformed[2]
Transformed fragment.
Definition: gemm_shared_stream.h:128
gemm_shared_tile.h
Defines iterators for efficiently loading and storing tiles to and from shared memory.
cutlass::gemm::SharedLoadStream
Definition: gemm_shared_stream.h:45
cutlass::gemm::SharedLoadStream::fragment
CUTLASS_DEVICE TransformedFragment const & fragment(int step=0) const
Returns the fragment for the given step.
Definition: gemm_shared_stream.h:114
cutlass::gemm::SharedLoadStream::transformer
Transformer transformer
The transformer.
Definition: gemm_shared_stream.h:126
cutlass::gemm::SharedLoadStream::Fragment
TransformedFragment Fragment
Make sure the fragments match.
Definition: gemm_shared_stream.h:57
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:46
static_assert
#define static_assert(__e, __m)
Definition: platform.h:153
cutlass::gemm::SharedLoadStream::Params::iterator
Iterator::Params iterator
The iterator params.
Definition: gemm_shared_stream.h:69
cutlass::gemm::SharedLoadStream::iterator
Iterator iterator
The iterator.
Definition: gemm_shared_stream.h:122
cutlass::gemm::SharedLoadStream::Params::initialize
CUTLASS_HOST_DEVICE int initialize()
Setup the params.
Definition: gemm_shared_stream.h:72
cutlass::gemm::SharedLoadStream::TransformedFragment
Transformer::OutputFragment TransformedFragment
The fragment that is obtained after the transformation by the transformer.
Definition: gemm_shared_stream.h:54
cutlass::gemm::SharedLoadStream::SharedLoadStream
CUTLASS_DEVICE SharedLoadStream(Params const &params, TensorRef const &ref)
Ctor.
Definition: gemm_shared_stream.h:82
cutlass::gemm::SharedLoadStream::fragment
CUTLASS_DEVICE TransformedFragment & fragment(int step=0)
Returns the fragment for the given step.
Definition: gemm_shared_stream.h:111
cutlass::gemm::SharedLoadStream::copy
CUTLASS_DEVICE void copy(int step)
Load the data from shared memory to the fetch fragment.
Definition: gemm_shared_stream.h:100
cutlass::gemm::SharedLoadStream::Params
The params.
Definition: gemm_shared_stream.h:67
cutlass::gemm::SharedLoadStream::initialize
CUTLASS_DEVICE void initialize(Params const &params, TensorRef const &ref)
Initialize the stream.
Definition: gemm_shared_stream.h:87
cutlass::gemm::SharedLoadStream::Scalar
Iterator::Scalar Scalar
Scalar data type.
Definition: gemm_shared_stream.h:61
cutlass::gemm::SharedLoadStream::FetchedFragment
Iterator::Fragment FetchedFragment
The fragment that is copied from shared memory.
Definition: gemm_shared_stream.h:52
cutlass::gemm::SharedLoadStream::Transformer
Transformer_ Transformer
The transformer.
Definition: gemm_shared_stream.h:49
cutlass::gemm::SharedLoadStream::Iterator
Iterator_ Iterator
The load iterator.
Definition: gemm_shared_stream.h:47
cutlass::gemm::SharedLoadStream::copy
CUTLASS_DEVICE void copy()
Load the data from shared memory to the fetch fragment.
Definition: gemm_shared_stream.h:95