Skip to content

Commit a29c799

Browse files
dancasarindancasarin
committed
True out-of-place ngfft
1 parent bf2fe73 commit a29c799

7 files changed

Lines changed: 225 additions & 118 deletions

File tree

‎include/kfr/dft/fft.hpp‎

Lines changed: 94 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1505,6 +1505,23 @@ size_t ngfft_twiddle_count(ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>);
15051505
template <typename T, dft_algorithm algo>
15061506
bool ngfft_initialize(ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>);
15071507

1508+
/**
1509+
* @brief Executes the FFT with separate input and output buffers.
1510+
*
1511+
* The transform reads from the input buffer and writes the result to the output
1512+
* buffer. No scaling is applied.
1513+
*
1514+
* @tparam T Floating-point scalar type.
1515+
* @tparam algo FFT algorithm.
1516+
* @tparam inverse If true, performs the inverse FFT; otherwise the forward FFT.
1517+
* @param plan The initialized plan.
1518+
* @param out Output buffer of `2^plan.l2fftsize` complex elements.
1519+
* @param in Input buffer of `2^plan.l2fftsize` complex elements.
1520+
*/
1521+
template <typename T, dft_algorithm algo, bool inverse>
1522+
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>, complex<T>* out,
1523+
const complex<T>* in);
1524+
15081525
/**
15091526
* @brief Executes the FFT in-place on the given complex buffer.
15101527
*
@@ -1518,25 +1535,48 @@ bool ngfft_initialize(ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>);
15181535
* @param inout Input/output buffer of `2^plan.l2fftsize` complex elements.
15191536
*/
15201537
template <typename T, dft_algorithm algo, bool inverse>
1521-
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>,
1522-
complex<T>* inout);
1538+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>,
1539+
complex<T>* inout)
1540+
{
1541+
return ngfft_execute<T, algo, inverse>(plan, cval<dft_algorithm, algo>, cbool_t<inverse>(), inout, inout);
1542+
}
15231543

15241544
/**
15251545
* @brief Executes the FFT in-place with a runtime direction flag.
15261546
* @copydetails ngfft_execute(const ngfft_plan<T>&, cval_t<dft_algorithm,algo>, cbool_t<inverse>, complex<T>*)
15271547
* @param inverse If true, performs the inverse FFT; otherwise the forward FFT.
15281548
*/
15291549
template <typename T, dft_algorithm algo>
1530-
inline void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, bool inverse,
1531-
complex<T>* inout)
1550+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, bool inverse,
1551+
complex<T>* inout)
1552+
{
1553+
if (inverse)
1554+
{
1555+
return ngfft_execute<T, algo, true>(plan, cval<dft_algorithm, algo>, ctrue, inout, inout);
1556+
}
1557+
else
1558+
{
1559+
return ngfft_execute<T, algo, false>(plan, cval<dft_algorithm, algo>, cfalse, inout, inout);
1560+
}
1561+
}
1562+
1563+
/**
1564+
* @brief Executes the FFT with separate input and output buffers and a runtime direction flag.
1565+
* @copydetails ngfft_execute(const ngfft_plan<T>&, cval_t<dft_algorithm,algo>, cbool_t<inverse>, complex<T>*,
1566+
* const complex<T>*)
1567+
* @param inverse If true, performs the inverse FFT; otherwise the forward FFT.
1568+
*/
1569+
template <typename T, dft_algorithm algo>
1570+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, bool inverse,
1571+
complex<T>* out, const complex<T>* in)
15321572
{
15331573
if (inverse)
15341574
{
1535-
return ngfft_execute<T, algo, true>(plan, cval<dft_algorithm, algo>, ctrue, inout);
1575+
return ngfft_execute<T, algo, true>(plan, cval<dft_algorithm, algo>, ctrue, out, in);
15361576
}
15371577
else
15381578
{
1539-
return ngfft_execute<T, algo, false>(plan, cval<dft_algorithm, algo>, cfalse, inout);
1579+
return ngfft_execute<T, algo, false>(plan, cval<dft_algorithm, algo>, cfalse, out, in);
15401580
}
15411581
}
15421582

@@ -1585,13 +1625,36 @@ inline bool ngfft_initialize(ngfft_plan<T>& plan, dft_algorithm algo = dft_algor
15851625
* @param algo FFT algorithm (defaults to `fourstep`).
15861626
*/
15871627
template <typename T, bool inverse>
1588-
inline void ngfft_execute(const ngfft_plan<T>& plan, cbool_t<inverse>, complex<T>* inout,
1589-
dft_algorithm algo = dft_algorithm::fourstep)
1628+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, cbool_t<inverse>, complex<T>* inout,
1629+
dft_algorithm algo = dft_algorithm::fourstep)
15901630
{
15911631
switch (algo)
15921632
{
15931633
case dft_algorithm::fourstep:
1594-
return ngfft_execute(plan, cval<dft_algorithm, dft_algorithm::fourstep>, cbool_t<inverse>(), inout);
1634+
return ngfft_execute(plan, cval<dft_algorithm, dft_algorithm::fourstep>, cbool_t<inverse>(), inout,
1635+
inout);
1636+
default:
1637+
KFR_UNREACHABLE;
1638+
}
1639+
}
1640+
1641+
/**
1642+
* @brief Executes the FFT with separate input and output buffers, a compile-time direction and runtime
1643+
* algorithm.
1644+
* @tparam inverse If true, performs the inverse FFT; otherwise the forward FFT.
1645+
* @param plan The initialized plan.
1646+
* @param out Output buffer of `2^plan.l2fftsize` complex elements.
1647+
* @param in Input buffer of `2^plan.l2fftsize` complex elements.
1648+
* @param algo FFT algorithm (defaults to `fourstep`).
1649+
*/
1650+
template <typename T, bool inverse>
1651+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, cbool_t<inverse>, complex<T>* out,
1652+
const complex<T>* in, dft_algorithm algo = dft_algorithm::fourstep)
1653+
{
1654+
switch (algo)
1655+
{
1656+
case dft_algorithm::fourstep:
1657+
return ngfft_execute(plan, cval<dft_algorithm, dft_algorithm::fourstep>, cbool_t<inverse>(), out, in);
15951658
default:
15961659
KFR_UNREACHABLE;
15971660
}
@@ -1605,13 +1668,31 @@ inline void ngfft_execute(const ngfft_plan<T>& plan, cbool_t<inverse>, complex<T
16051668
* @param algo FFT algorithm (defaults to `fourstep`).
16061669
*/
16071670
template <typename T>
1608-
inline void ngfft_execute(const ngfft_plan<T>& plan, bool inverse, complex<T>* inout,
1609-
dft_algorithm algo = dft_algorithm::fourstep)
1671+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, bool inverse, complex<T>* inout,
1672+
dft_algorithm algo = dft_algorithm::fourstep)
1673+
{
1674+
if (inverse)
1675+
return ngfft_execute(plan, ctrue, inout, inout, algo);
1676+
else
1677+
return ngfft_execute(plan, cfalse, inout, inout, algo);
1678+
}
1679+
1680+
/**
1681+
* @brief Executes the FFT with separate input and output buffers, runtime direction and algorithm.
1682+
* @param plan The initialized plan.
1683+
* @param inverse If true, performs the inverse FFT; otherwise the forward FFT.
1684+
* @param out Output buffer of `2^plan.l2fftsize` complex elements.
1685+
* @param in Input buffer of `2^plan.l2fftsize` complex elements.
1686+
* @param algo FFT algorithm (defaults to `fourstep`).
1687+
*/
1688+
template <typename T>
1689+
KFR_INLINE void ngfft_execute(const ngfft_plan<T>& plan, bool inverse, complex<T>* out, const complex<T>* in,
1690+
dft_algorithm algo = dft_algorithm::fourstep)
16101691
{
16111692
if (inverse)
1612-
return ngfft_execute(plan, ctrue, inout, algo);
1693+
return ngfft_execute(plan, ctrue, out, in, algo);
16131694
else
1614-
return ngfft_execute(plan, cfalse, inout, algo);
1695+
return ngfft_execute(plan, cfalse, out, in, algo);
16151696
}
16161697

16171698
#ifdef KFR_CLASSIC_FFT

‎src/dft/dft.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ KFR_MULTI_PROTO(namespace impl {
156156
bool ngfft_initialize(ngfft_plan<T> & plan, cval_t<dft_algorithm, algo>);
157157
template <typename T, dft_algorithm algo, bool inverse>
158158
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>,
159-
complex<T>* inout);
159+
complex<T>* out, const complex<T>* in);
160160
})
161161

162162
#ifdef KFR_CLASSIC_FFT
@@ -251,10 +251,10 @@ bool ngfft_initialize(ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>)
251251
KFR_MULTI_GATE(return ns::impl::ngfft_initialize(plan, cval<dft_algorithm, algo>));
252252
}
253253
template <typename T, dft_algorithm algo, bool inverse>
254-
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>,
255-
complex<T>* inout)
254+
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>, complex<T>* out,
255+
const complex<T>* in)
256256
{
257-
KFR_MULTI_GATE(ns::impl::ngfft_execute(plan, cval<dft_algorithm, algo>, cbool<inverse>, inout));
257+
KFR_MULTI_GATE(ns::impl::ngfft_execute(plan, cval<dft_algorithm, algo>, cbool<inverse>, out, in));
258258
}
259259

260260
template size_t ngfft_twiddle_count<float, dft_algorithm::fourstep>(
@@ -270,17 +270,18 @@ template bool ngfft_initialize<double, dft_algorithm::fourstep>(
270270
ngfft_plan<double>&, cval_t<dft_algorithm, dft_algorithm::fourstep>);
271271

272272
template void ngfft_execute<float, dft_algorithm::fourstep, false>(
273-
const ngfft_plan<float>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<false>,
274-
complex<float>*);
273+
const ngfft_plan<float>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<false>, complex<float>*,
274+
const complex<float>*);
275275
template void ngfft_execute<float, dft_algorithm::fourstep, true>(
276-
const ngfft_plan<float>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<true>, complex<float>*);
276+
const ngfft_plan<float>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<true>, complex<float>*,
277+
const complex<float>*);
277278

278279
template void ngfft_execute<double, dft_algorithm::fourstep, false>(
279280
const ngfft_plan<double>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<false>,
280-
complex<double>*);
281+
complex<double>*, const complex<double>*);
281282
template void ngfft_execute<double, dft_algorithm::fourstep, true>(
282283
const ngfft_plan<double>&, cval_t<dft_algorithm, dft_algorithm::fourstep>, cbool_t<true>,
283-
complex<double>*);
284+
complex<double>*, const complex<double>*);
284285

285286
#endif
286287

‎src/dft/fft-impl.hpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ template <typename T, dft_algorithm algo>
6666
bool ngfft_initialize(ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>);
6767

6868
template <typename T, dft_algorithm algo, bool inverse>
69-
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>,
70-
complex<T>* inout);
69+
void ngfft_execute(const ngfft_plan<T>& plan, cval_t<dft_algorithm, algo>, cbool_t<inverse>, complex<T>* out,
70+
const complex<T>* in);
7171

7272
} // namespace impl
7373

@@ -329,10 +329,8 @@ struct fft_ng_stage_impl : dft_stage<T>
329329
template <bool inverse>
330330
KFR_MEM_INTRINSIC void do_execute(complex<T>* out, const complex<T>* in, u8*)
331331
{
332-
if (in != out) [[unlikely]]
333-
builtin_memcpy(out, in, sizeof(complex<T>) * this->stage_size);
334332
ngfft_plan<T> plan{ uint8_t(this->user), ptr_cast<complex<T>>(this->data) };
335-
impl::ngfft_execute(plan, cval<dft_algorithm, algo>, cbool_t<inverse>(), out);
333+
impl::ngfft_execute(plan, cval<dft_algorithm, algo>, cbool_t<inverse>(), out, in);
336334
}
337335
};
338336

‎src/dft/ft.hpp‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2956,7 +2956,7 @@ struct bfly_parallel_bfly_base<Radix, T, N, bfly_twiddles_type::vector>
29562956
};
29572957

29582958
template <size_t Radix, typename T, size_t N, bool inverse, bfly_twiddles_type twiddles, dft_decomp decomp,
2959-
bool in_split, bool out_split, size_t prefetch = 0>
2959+
bool in_split, bool out_split, size_t prefetch = 0, bool inplace = true>
29602960
struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
29612961
{
29622962
constexpr static bool split_format = in_split || out_split;
@@ -2975,12 +2975,23 @@ struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
29752975
}
29762976
}
29772977

2978-
complex<T>* inout;
2978+
complex<T>* out;
2979+
const complex<T>* in;
29792980
size_t stride;
29802981

2981-
KFR_MEM_INTRINSIC bfly_parallel_bfly(complex<T>* inout, size_t stride, const std::complex<T>* tw)
2982+
complex<T>* get_out() const noexcept { return out; }
2983+
const complex<T>* get_in() const noexcept
2984+
{
2985+
if constexpr (inplace)
2986+
return out;
2987+
else
2988+
return in;
2989+
}
2990+
2991+
KFR_MEM_INTRINSIC bfly_parallel_bfly(complex<T>* out, const complex<T>* in, size_t stride,
2992+
const std::complex<T>* tw)
29822993
requires(twiddles != bfly_twiddles_type::none)
2983-
: inout(inout), stride(stride)
2994+
: out(out), in(in), stride(stride)
29842995
{
29852996
if constexpr (twiddles == bfly_twiddles_type::scalar)
29862997
{
@@ -2993,9 +3004,9 @@ struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
29933004
this->tw = tw;
29943005
}
29953006
}
2996-
KFR_MEM_INTRINSIC bfly_parallel_bfly(complex<T>* inout, size_t stride)
3007+
KFR_MEM_INTRINSIC bfly_parallel_bfly(complex<T>* out, const complex<T>* in, size_t stride)
29973008
requires(twiddles == bfly_twiddles_type::none)
2998-
: inout(inout), stride(stride)
3009+
: out(out), in(in), stride(stride)
29993010
{
30003011
}
30013012

@@ -3041,25 +3052,26 @@ struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
30413052
cvec<T, N> w;
30423053
if constexpr (I == 0 && twiddles != bfly_twiddles_type::matrix)
30433054
{
3044-
w = deinterleave(cread_prefetch<N, prefetch>(this->inout));
3055+
w = deinterleave(cread_prefetch<N, prefetch>(this->get_in()));
30453056
}
30463057
else if constexpr (decomp == dft_decomp::dit)
30473058
{
30483059
constexpr size_t J = br(I);
30493060
if constexpr (twiddles != bfly_twiddles_type::none)
30503061
{
3051-
w = cmuli<inverse>(cbool<split_format>,
3052-
deinterleave(cread_prefetch<N, prefetch>(this->inout + J * this->stride)),
3053-
get_tw<I>());
3062+
w = cmuli<inverse>(
3063+
cbool<split_format>,
3064+
deinterleave(cread_prefetch<N, prefetch>(this->get_in() + J * this->stride)),
3065+
get_tw<I>());
30543066
}
30553067
else
30563068
{
3057-
w = deinterleave(cread_prefetch<N, prefetch>(this->inout + J * this->stride));
3069+
w = deinterleave(cread_prefetch<N, prefetch>(this->get_in() + J * this->stride));
30583070
}
30593071
}
30603072
else
30613073
{
3062-
w = deinterleave(cread_prefetch<N, prefetch>(this->inout + I * this->stride));
3074+
w = deinterleave(cread_prefetch<N, prefetch>(this->get_in() + I * this->stride));
30633075
}
30643076
return w;
30653077
}
@@ -3069,24 +3081,24 @@ struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
30693081
{
30703082
if constexpr (I == 0 && twiddles != bfly_twiddles_type::matrix)
30713083
{
3072-
cwrite<N, false>(this->inout, interleave(w));
3084+
cwrite<N, false>(this->get_out(), interleave(w));
30733085
}
30743086
else if constexpr (decomp == dft_decomp::dif)
30753087
{
30763088
constexpr size_t J = br(I);
30773089
if constexpr (twiddles != bfly_twiddles_type::none)
30783090
{
3079-
cwrite<N, false>(this->inout + J * this->stride,
3091+
cwrite<N, false>(this->get_out() + J * this->stride,
30803092
interleave(cmuli<inverse>(cbool<split_format>, w, get_tw<I>())));
30813093
}
30823094
else
30833095
{
3084-
cwrite<N, false>(this->inout + J * this->stride, interleave(w));
3096+
cwrite<N, false>(this->get_out() + J * this->stride, interleave(w));
30853097
}
30863098
}
30873099
else
30883100
{
3089-
cwrite<N, false>(this->inout + I * this->stride, interleave(w));
3101+
cwrite<N, false>(this->get_out() + I * this->stride, interleave(w));
30903102
}
30913103
}
30923104

@@ -3119,7 +3131,12 @@ struct bfly_parallel_bfly : bfly_parallel_bfly_base<Radix, T, N, twiddles>
31193131

31203132
KFR_INLINE_MEMBER void begin() noexcept {}
31213133
KFR_INLINE_MEMBER void end() noexcept {}
3122-
KFR_INLINE_MEMBER void advance() noexcept { this->inout += N; }
3134+
KFR_INLINE_MEMBER void advance() noexcept
3135+
{
3136+
if constexpr (!inplace)
3137+
this->in += N;
3138+
this->out += N;
3139+
}
31233140
};
31243141

31253142
} // namespace intr

0 commit comments

Comments
 (0)