/***************************************************************************
 *   Copyright (C) 2011,2012 by                                            *
 *   Oliver Bock oliver.bock[AT]aei.mpg.de                                 *
 *   Heinz-Bernd Eggenstein heinz-bernd.eggenstein[AT]aei.mpg.de           *
 *                                                                         *
 *   This file is part of Einstein@Home (Radio Pulsar Edition).            *
 *                                                                         *
 *   Description:                                                          *
 *                                                                         *
 *   Einstein@Home is free software: you can redistribute it and/or modify *
 *   it under the terms of the GNU General Public License as published     *
 *   by the Free Software Foundation, version 2 of the License.            *
 *                                                                         *
 *   Einstein@Home is distributed in the hope that it will be useful,      *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the          *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with Einstein@Home. If not, see <http://www.gnu.org/licenses/>. *
 *                                                                         *
 ***************************************************************************/

#ifndef DEMOD_BINARY_OCL_KERNELS_H
#define DEMOD_BINARY_OCL_KERNELS_H

#define OCL_RESAMP_OFFSETS_WGSIZE_X     256
#define OCL_RESAMP_WGSIZE_X             256
#define OCL_RESAMP_REDUCTION_WGSIZE_X   256
#define OCL_RESAMP_PADDING_WGSIZE_X     256
#define OCL_PS_WGSIZE_X                 256
#define OCL_PS_R3_R2C_WGSIZE_X          256
#define OCL_BUFFER_FILL_WGSIZE_X        256
#define OCL_HS_WGSIZE_X                 256

// TODO: check: changes needed below?

// watch out for spaces right before each line break!
static char defaultCompileOptions[] =
"-Werror "
"-cl-single-precision-constant "
"%s "
"-D OCL_HS_CONDITIONAL_STORES_THRESHOLD=1 "
"-D OCL_RESAMP_REDUCTION_WGSIZE_X=%i "
"-D OCL_PS_WGSIZE_X=%i "
"-D OCL_PS_R3_R2C_WGSIZE_X=%i "
"-D OCL_HS_WGSIZE_X=%i "
"-D OCL_HS_LOG_WGSIZE_X=%i "
"-D OCL_LOG_PS_PAGE_SIZE_X=%i";


static const char deviceSinLUTLookup[] =
"#define ERP_SINCOS_LUT_RES    64 \n"
"#define ERP_TWO_PI            6.283185f \n"
""
"float deviceSinLUTLookup(__constant float *, __constant float *, __private float); \n"
""
"float deviceSinLUTLookup(__constant float *constSinSamples, \n"
"                         __constant float *constCosSamples, \n"
"                         __private  float  x) \n"
"{ \n"
"    float xt; \n"
"    int i0; \n"
"    float d, d2; \n"
"    float ts, tc; \n"
""
"    /* normalize value */ \n"
"    xt = modf(x / ERP_TWO_PI, &x); /* xt in (-1, 1) */ \n"
"    if ( xt < 0.0f ) { \n"
"        xt += 1.0f;                /* xt in [0, 1 ) */ \n"
"    } \n"
""
"    /* determine LUT index */ \n"
"    i0 = (int) (xt * ERP_SINCOS_LUT_RES + 0.5f); \n"
"    d = d2 = ERP_TWO_PI * (xt - 1.0f / ERP_SINCOS_LUT_RES * i0); \n"
"    d2 *= 0.5f * d; \n"
""
"    /* fetch sin/cos samples from constant memory */ \n"
"    ts = constSinSamples[i0]; \n"
"    tc = constCosSamples[i0]; \n"
""
"    /* use taylor-expansion for sin around samples */ \n"
"    return ts + d * tc - d2 * ts; \n"
"} \n";


static const char kernelTimeSeriesModulation[] =
"__kernel void kernelTimeSeriesModulation(__constant float *constSinSamples, \n"
"                                         __constant float *constCosSamples, \n"
"                                         __private  float  tau, \n"
"                                         __private  float  Omega, \n"
"                                         __private  float  Psi0, \n"
"                                         __private  float  dt, \n"
"                                         __private  float  step_inv, \n"
"                                         __private  float  S0, \n"
"                                         __global   float *del_t) \n"
"{ \n"
"    size_t i = get_global_id(0); \n"
""
"    /* compute time offset */ \n"
"    float t = i * dt; \n"
"    float x = Omega * t + Psi0; \n"
"    float sinX = deviceSinLUTLookup(constSinSamples, constCosSamples, x); \n"
""
"    /* compute time offsets */ \n"
"    del_t[i] = tau * sinX * step_inv - S0; \n"
"} \n";


static const char kernelTimeSeriesLengthModulated[] =
"__kernel void kernelTimeSeriesLengthModulated(__private unsigned int  nsamples_unpadded, \n"
"                                              __global  float        *del_t, \n"
"                                              __global  unsigned int *timeSeriesLength) \n"
"{ \n"
"    /* number of timesteps that fit into the duration = at most the amount we had before */ \n"
"    size_t n_steps = nsamples_unpadded - 1; \n"
""
"    /* TODO: avoid global memory reads!!! */ \n"
"    /* nearest_idx (see time_series_resampling kernel) must not exceed n_unpadded - 1, so go back as far as needed to ensure that */ \n"
"    while(n_steps - del_t[n_steps] >= nsamples_unpadded - 1) { \n"
"        n_steps--; \n"
"    } \n"
""
"    /* copy length into global variable */ \n"
"    *timeSeriesLength = n_steps; \n"
"} \n";


static const char kernelTimeSeriesResampling[] =
"__kernel void kernelTimeSeriesResampling(__global  float        *input, \n"
"                                         __global  float        *del_t, \n"
"                                         __private unsigned int  length, \n"
"                                         __global  float        *output) \n"
"{ \n"
"    size_t i = get_global_id(0); \n"
""
"    /* TODO: ensure coalesced memory access (load/store) !!! */ \n"
"    /* only resample 'existing' time samples */ \n"
"    if(i < length) { \n"
"        /* sample i arrives at the detector at i - del_t[i], choose nearest neighbor */ \n"
"        int nearest_idx = (int)(i - del_t[i] + 0.5f); \n"
""
"        /* set i-th bin in resampled time series (at the pulsar) to nearest_idx bin from de-dispersed time series */ \n"
"        output[i] = input[nearest_idx]; \n"
"    } \n"
"    else { \n"
"        /* set remaining buffercells to zero (for upcoming sum reduction) */ \n"
"        output[i] = 0.0f; \n"
"    } \n"
"} \n";


static const char kernelTimeSeriesMeanReduction[] =
"__kernel void kernelTimeSeriesMeanReduction(__global float *input, \n"
"                                            __global float *output,\n"
"                                            __private unsigned int input_offset,\n"
"                                            __private unsigned int output_offset) \n"
"{ \n"
"    __local float sharedPartialSum[OCL_RESAMP_REDUCTION_WGSIZE_X]; \n"
""
"    size_t i = get_global_id(0); \n"
"    size_t workItemIdx_x = get_local_id(0); \n"
""
"    /* coalesced load of time series data into shared memory */ \n"
"    sharedPartialSum[workItemIdx_x] = input[i+input_offset]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    /* compute sum of current workgroup (in log2(workgroup size) iterations) */ \n"
"    for(size_t stride = get_local_size(0) >> 1; stride > 0; stride >>= 1) { \n"
"        /* sum two strided values per thread */ \n"
"        if(workItemIdx_x < stride) { \n"
"            sharedPartialSum[workItemIdx_x] += sharedPartialSum[workItemIdx_x + stride]; \n"
"        } \n"
""
"        /* wait for (partial) block summing iteration to finish */ \n"
"        barrier(CLK_LOCAL_MEM_FENCE); \n"
"    } \n"
""
"    /* store sum of current workgroup in global memory (single thread) */ \n"
"    if(workItemIdx_x == 0) { \n"
"        output[get_group_id(0)+output_offset] = sharedPartialSum[0]; \n"
"    } \n"
"} \n";


static const char kernelTimeSeriesPadding[] =
"__kernel void kernelTimeSeriesPadding(__private float         mean, \n"
"                                      __private unsigned int  offset, \n"
"                                      __global  float        *input,  \n"
"                                      __global  float        *output) \n"
"{ \n"
"    size_t i= get_global_id(0); \n"
""
"    /* can't be avoided as time series varies in length (incl. non-multiple-of-32 values) */ \n"
"    if(i >= offset) { \n"
"        /* coalesced store of resampled time series padding data to global memory */ \n"
"        output[i] = mean; \n"
"    } else {              \n"
"        output[i]=input[i];\n"
"    } \n"
"} \n";




// each thread takes care to padd and transpose k many values
// (where padded time series length is N=k*2^n for some small k) (radix in argument list)
// this is hard to do in-situ, so we put the output in the same array at a defined offset,
// and zero out the input
// Note that input is interpreted as complex numbers in interleavd format, 
// however input length might be odd, so here we shouldn't use float2 as the last element might need
// special treatment ("mean padding" of imag. part).
// TODO: optimize

static const char kernelTimeSeriesPaddingTranspose[] =
"__kernel void kernelTimeSeriesPaddingTranspose(__private float         mean, \n"
"                                      __private unsigned int  padding_offset, \n"
"                                      __private unsigned int  radix,       \n"
"                                      __private unsigned int  subFFTlen,   \n"
"                                      __global  float         *input , \n"  
"                                      __global  float         *output) \n"
"{ \n"
"    size_t id1 = get_global_id(0); \n"
"    size_t i,k,folded_i; \n"
"    float v;   \n"
""
"    for(i=radix*2*id1, folded_i=2*id1,k=0; k < radix ; i+=2, k++, folded_i+=subFFTlen) { \n"
"    //real part    \n"
"      v= (i >= padding_offset) ? mean : input[i]; \n"  
"      output[folded_i] = v; \n"
"    // imag part \n"
"      v= (i+1 >= padding_offset) ? mean : input[i+1]; \n"  
"      output[folded_i+1] = v; \n"
"    } \n"
"} \n";


static const char kernelPowerSpectrum[] =
"#define complexMul(a,b) ((float2)(-((a).y)*(b).y + (a).x * (b).x, (a).y* (b).x + (a).x * (b).y))\n"
"\n"
"#define untangle(res,Ha,Hb,tf) \\\n"         
"   { float2 _t1= (float2) (0.5f * ((Ha).x + (Hb).x) ,0.5f*((Ha).y - (Hb).y));     \\\n"
"     float2 _t2= (float2) (0.5f * ((Ha).y + (Hb).y), 0.5f*(-(Ha).x + (Hb).x ));   \\\n"
"     (res) = _t1 + complexMul(_t2,(tf)); \\\n"
"   } \n"
""
"#define untangle2(res,Ha,Hb,tf) \\\n"         
"   { float2 _t1= (float2) (1.0f + tf.y , -tf.x);     \\\n"
"     float2 _t2= (float2) (1.0f - tf.y, tf.x);   \\\n"
"     float2 _Hbconj = (float2) (Hb.x,-Hb.y);     \\\n"
"     (res) = 0.5f * (complexMul(_t1,Ha) + complexMul(_t2,_Hbconj)); \\\n"
"   } \n"
""
""
""
"__kernel void kernelPowerSpectrum(__global  float2 *       fft_complex_data, \n"
"                                  __global  float *        twiddle_data,  \n"
"                                  __private float          norm_factor, \n"
"                                  __private unsigned int   complexFFTlength,\n" 
"                                  __global  float  *       ps_data) \n"
"{ \n"
"    __local float2 localFFTData[OCL_PS_WGSIZE_X]; \n"
"    __local float2 localFFTDataM[OCL_PS_WGSIZE_X]; \n"
"    __local float2 localFFTTwiddleData[OCL_PS_WGSIZE_X]; \n"
"    __local float2 localFFTTwiddleDataM[OCL_PS_WGSIZE_X]; \n"
""
"    float2 res,resM;                         \n"
"    size_t globalId = get_global_id(0); \n"
"    size_t localId = get_local_id(0); \n"
"    size_t reverseIdx = (globalId==0) ? 0 : complexFFTlength - globalId; \n"
""
"    int ignore = (globalId +globalId > complexFFTlength);\n"
""
"    /* coalesced load of FFT data into local memory */ \n"
"    localFFTData[localId] = fft_complex_data[globalId]; \n"
"    localFFTDataM[localId] = fft_complex_data[reverseIdx]; \n"
// TODO optimize to prevent reverse order of reading (wrt thread Idx)
"    localFFTTwiddleData[localId].x  = twiddle_data[globalId]; \n"
"    localFFTTwiddleData[localId].y  = twiddle_data[globalId+complexFFTlength]; \n"
"    localFFTTwiddleDataM[localId].x = twiddle_data[reverseIdx]; \n"
"    localFFTTwiddleDataM[localId].y = twiddle_data[reverseIdx+complexFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
// untangling real FFT result from DFT of fake complex input data
"    untangle(res,localFFTData[localId],localFFTDataM[localId],localFFTTwiddleData[localId]); \n"
"    untangle(resM,localFFTDataM[localId],localFFTData[localId],localFFTTwiddleDataM[localId]); \n"
""
"    /* computer power spectrum */ \n"
"    if(0 == globalId ) { \n"
"       ps_data[0] = 0.0f; \n"
"       untangle(res,localFFTData[localId],localFFTData[localId],(float2) (-1.0f,0.0f)); \n"
"       ps_data[complexFFTlength]= norm_factor * (res.x * res.x + res.y * res.y);\n"
"    } \n"
"    else { \n"
"       if (!ignore) ps_data[globalId] = norm_factor * (res.x * res.x + res.y * res.y); \n"
"       if (!ignore) ps_data[reverseIdx] = norm_factor * (resM.x * resM.x + resM.y * resM.y); \n"
"    }\n"
"} \n";



// modified powerspectum kernel that does the following things in sequence:
//      - multiply input array with (complex!) twiddle factors (3 multiplications)
//      - perform a slightly optimzed radix 3 butterfly step   (2 multiplications)
//      - reassemble complex DFT when the input was real but put into a half-length
//        complex2complex FFT
//      - compute the power and write result to output buffer
//      Note that data is in interleaved format in this case

static const char kernelPowerSpectrum_radix3_r2c[] =
"#define complexMul(a,b) ((float2)(-((a).y)*(b).y + (a).x * (b).x, (a).y* (b).x + (a).x * (b).y))\n"
"\n"
"#define untangle(res,Ha,Hb,tf) \\\n"         
"   { float2 _t1= (float2) (0.5f * ((Ha).x + (Hb).x) ,0.5f*((Ha).y - (Hb).y));     \\\n"
"     float2 _t2= (float2) (0.5f * ((Ha).y + (Hb).y), 0.5f*(-(Ha).x + (Hb).x ));   \\\n"
"     (res) = _t1 + complexMul(_t2,(tf)); \\\n"
"   } \n"
""
"#define untangle2(res,Ha,Hb,tf) \\\n"         
"   { float2 _t1= (float2) (1.0f + tf.y , -tf.x);     \\\n"
"     float2 _t2= (float2) (1.0f - tf.y, tf.x);   \\\n"
"     float2 _Hbconj = (float2) (Hb.x,-Hb.y);     \\\n"
"     (res) = 0.5f * (complexMul(_t1,Ha) + complexMul(_t2,_Hbconj)); \\\n"
"   } \n"

"__kernel void kernelPowerSpectrum_radix3_r2c(__global  float2        *fft_data, \n"
"                                         __global  float        *fft_twiddle_data, \n"
"                                         __global  float        *fft_twiddle_data_rc, \n"
"                                         __private int           subFFTlength, \n"
"                                         __private unsigned int  ps_length, \n"
"                                         __private float         twiddle_factor, \n"
"                                         __private float         norm_factor, \n"
"                                         __global  float        *ps_data) \n"
"{ \n"
"    __local float2 localFFTData_mod0[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localFFTData_mod1[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localFFTData_mod2[OCL_PS_R3_R2C_WGSIZE_X]; \n"
""
"    __local float2 localTwiddle_mod0[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localTwiddle_mod1[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localTwiddle_mod2[OCL_PS_R3_R2C_WGSIZE_X]; \n"
""
"    __local float2 localFFTData_modM0[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localFFTData_modM1[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localFFTData_modM2[OCL_PS_R3_R2C_WGSIZE_X]; \n"
""
"    __local float2 localTwiddle_modM0[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localTwiddle_modM1[OCL_PS_R3_R2C_WGSIZE_X]; \n"
"    __local float2 localTwiddle_modM2[OCL_PS_R3_R2C_WGSIZE_X]; \n"
""
"    //float  ac1,ac2,bd1,bd2;\n"
"    //float2 prod_1,prod_2; \n"
"    float2 r0_1;             \n"   
"    float2 r1_1;             \n"
"    float2 r2_1;             \n"
"    float2 r0_2;             \n"   
"    float2 r1_2;             \n"
"    float2 r2_2;             \n"
"    float t1_1,t2_1,m_1;                 \n"
"    float t1_2,t2_2,m_2;                 \n"
"    size_t globalId = get_global_id(0); \n"
"    size_t localId = get_local_id(0); \n"
"    size_t    fullFFTlength= subFFTlength+subFFTlength+subFFTlength; \n"
"    size_t mask =  subFFTlength -1; \n"
"    size_t index_reverse = (subFFTlength-globalId) & mask;  \n"
"    "
"    int ignore=(globalId > (size_t) subFFTlength /2) ;\n" 
""
"    /* coalesced load of FFT data into local memory */ \n"
"    localFFTData_mod0[localId] = fft_data[globalId]; \n"
"    localFFTData_mod1[localId] = fft_data[globalId+subFFTlength]; \n"
"    localFFTData_mod2[localId] = fft_data[globalId+subFFTlength+subFFTlength]; \n"
""
"" // TODO opt: avoid reading stuff in reverse order wrt thread index ??
"" // TODO opt: better interleave computing and loading data 
""
"    /* coalesced load of FFT data into local memory */ \n"
"    localFFTData_modM0[localId] = fft_data[index_reverse]; \n"
"    localFFTData_modM1[localId] = fft_data[index_reverse+subFFTlength]; \n"
"    localFFTData_modM2[localId] = fft_data[index_reverse+subFFTlength+subFFTlength]; \n"
""
""
" //   localTwiddle_mod0[localId].x = fft_twiddle_data[globalId]; \n"
" //   localTwiddle_mod0[localId].y = fft_twiddle_data[globalId+fullFFTlength]; \n"
"    localTwiddle_mod1[localId].x = fft_twiddle_data[globalId+subFFTlength]; \n"
"    localTwiddle_mod1[localId].y = fft_twiddle_data[globalId+subFFTlength+fullFFTlength]; \n"
"    localTwiddle_mod2[localId].x = fft_twiddle_data[globalId+subFFTlength+subFFTlength]; \n"
"    localTwiddle_mod2[localId].y = fft_twiddle_data[globalId+subFFTlength+subFFTlength+fullFFTlength]; \n"
""
" //   localTwiddle_modM0[localId].x = fft_twiddle_data[index_reverse]; \n"
" //   localTwiddle_modM0[localId].y = fft_twiddle_data[index_reverse+fullFFTlength]; \n"
"    localTwiddle_modM1[localId].x = fft_twiddle_data[index_reverse+subFFTlength]; \n"
"    localTwiddle_modM1[localId].y = fft_twiddle_data[index_reverse+subFFTlength+fullFFTlength]; \n"
"    localTwiddle_modM2[localId].x = fft_twiddle_data[index_reverse+subFFTlength+subFFTlength]; \n"
"    localTwiddle_modM2[localId].y = fft_twiddle_data[index_reverse+subFFTlength+subFFTlength+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    // complex multiply of twiddle factor \n"
"    //TODO: vectorize                                          \n"
" // NOTHING to do for first localFFTData_mod0 and localFFTData_modM0 since twiddle factor \n "
" // is always 1.0 + 0i for them \n"
""
"    localFFTData_mod1[localId] = complexMul(localFFTData_mod1[localId],localTwiddle_mod1[localId])  ;\n"
"    localFFTData_modM1[localId] =  complexMul(localFFTData_modM1[localId],localTwiddle_modM1[localId]);\n"
""
""
"    localFFTData_mod2[localId] = complexMul(localFFTData_mod2[localId],localTwiddle_mod2[localId])  ;\n"
"    localFFTData_modM2[localId] = complexMul(localFFTData_modM2[localId],localTwiddle_modM2[localId]);\n"
"                                                                \n"
"                                                                \n"
"    // butterfly step                                           \n"
"    // Now compute (r0,r1,r2) = FFT(x0,x1,x2):                  \n"
"    // r_n = sum_k=0..2 (w^[n*k] x_k)   ,                       \n"
"    // where w:=e^(-2pi*i/3) = cos(2pi/3) - i*sin(2pi/3) = -1/2 - i*sqrt(3)/2       \n"
"    // which can be simplified to                               \n"
"    // (in MATLAB/octave notation)                              \n"
"    // a0 = real(x0); a1 = real(x1); a2 = real(x2);             \n"
"    // b0 = imag(x0); b1 = imag(x1); b2 = imag(x2);             \n"
"    // tf=sqrt(3)/2;                                            \n"
"    //                                                          \n"
"    // m1=(a1+a2)*0.5;                                          \n"
"    // m2=(b1-b2)*tf;                                           \n"
"    // m3=(b1+b2)*0.5;                                          \n"
"    // m4=(a1-a2)*tf;                                           \n"
"    //                                                          \n"
"    // res0= (a0+a1+a2) + i*(b0 + b1 + b2 );                    \n"
"    // res1= a0-m1 +m2  + i*(b0-m3 -m4);                        \n"
"    // res2= a0-m1 -m2  + i*(b0-m3 +m4);                        \n"
"    // However, we made sure through choice of the twiddle factors        \n"
"    // that x1 and x2 are already scaled by 0.5 before this step, so we can avoid   \n"
"    // the trivial multiplications by 0.5 here.                \n"
"    // With butterfly_twiddle_factor = sqrt(3),                \n"
"    // x0 = localFFTData_mod0, x1 = localFFTData_mod1, x2 = localFFTData_mod2,we get:         \n"
"    //                                                          \n"
"                                                                \n"
"    m_1= twiddle_factor * (localFFTData_mod1[localId].y - localFFTData_mod2[localId].y);        \n"
"    m_2= twiddle_factor * (localFFTData_modM1[localId].y - localFFTData_modM2[localId].y);        \n"
""
"                                                                \n"
"    t1_1=localFFTData_mod1[localId].x + localFFTData_mod2[localId].x;        \n"
"    t1_2=localFFTData_modM1[localId].x + localFFTData_modM2[localId].x;        \n"
""
"    r0_1.x = (t1_1+t1_1) + localFFTData_mod0[localId].x;             \n"
"    r0_2.x = (t1_2+t1_2) + localFFTData_modM0[localId].x;             \n"
""
"    t2_1= localFFTData_mod0[localId].x - t1_1;                      \n"
"    t2_2= localFFTData_modM0[localId].x - t1_2;                      \n"
""
"    r1_1.x = t2_1 +m_1;                                              \n"
"    r1_2.x = t2_2 +m_2;                                              \n"
""
"    r2_1.x = t2_1 -m_1;                                              \n"
"    r2_2.x = t2_2 -m_2;                                              \n"
""
""    
"                                                                \n"
"    m_1= twiddle_factor * (localFFTData_mod1[localId].x - localFFTData_mod2[localId].x);        \n"
"    m_2= twiddle_factor * (localFFTData_modM1[localId].x - localFFTData_modM2[localId].x);        \n"
""
"                                                                \n"
"    t1_1=localFFTData_mod1[localId].y + localFFTData_mod2[localId].y;        \n"
"    t1_2=localFFTData_modM1[localId].y + localFFTData_modM2[localId].y;        \n"
""
"    r0_1.y = (t1_1+t1_1) + localFFTData_mod0[localId].y;             \n"
"    r0_2.y = (t1_2+t1_2) + localFFTData_modM0[localId].y;             \n"
""
"    t2_1= localFFTData_mod0[localId].y - t1_1;                      \n"
"    t2_2= localFFTData_modM0[localId].y - t1_2;                      \n"
""
"    r1_1.y = t2_1 -m_1;                                              \n"
"    r1_2.y = t2_2 -m_2;                                              \n"
"    r2_1.y = t2_1 +m_1;                                              \n"
"    r2_2.y = t2_2 +m_2;                                              \n"
"                                                                \n"
""   // now take the result of the complex-to-complex transform and convert back to
""   // the DFT of the original problem, a Real to complex transform
""   // see Numerical Recipes, 3rd Edition, section 12.3.2, pp618-619
""
""   // first load the second set of twiddle factors for the combination step
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    localTwiddle_mod0[localId].x = fft_twiddle_data_rc[globalId]; \n"
"    localTwiddle_mod0[localId].y = fft_twiddle_data_rc[globalId+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    localTwiddle_mod1[localId].x = fft_twiddle_data_rc[globalId+subFFTlength]; \n"
"    localTwiddle_mod1[localId].y = fft_twiddle_data_rc[globalId+subFFTlength+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    localTwiddle_mod2[localId].x = fft_twiddle_data_rc[globalId+subFFTlength+subFFTlength]; \n"
"    localTwiddle_mod2[localId].y = fft_twiddle_data_rc[globalId+subFFTlength+subFFTlength+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    localTwiddle_modM0[localId].x = fft_twiddle_data_rc[subFFTlength-globalId]; \n"
"    localTwiddle_modM0[localId].y = fft_twiddle_data_rc[subFFTlength-globalId+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    localTwiddle_modM1[localId].x = fft_twiddle_data_rc[subFFTlength+subFFTlength-globalId]; \n"
"    localTwiddle_modM1[localId].y = fft_twiddle_data_rc[subFFTlength+subFFTlength-globalId+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
"    /* coalesced load of FFT twiddle factors into local memory */ \n"
"    /* Note: we have to avoid overrun for globalId==0  */ \n"
"    localTwiddle_modM2[localId].x = fft_twiddle_data_rc[(subFFTlength+subFFTlength+subFFTlength-globalId) % fullFFTlength]; \n"
"    localTwiddle_modM2[localId].y = fft_twiddle_data_rc[((subFFTlength+subFFTlength+subFFTlength-globalId) % fullFFTlength )+fullFFTlength]; \n"
""
"    /* wait for load to finish */ \n"
"    barrier(CLK_LOCAL_MEM_FENCE); \n"
""
""   
""   // we have three pairs of values (H_i, H_{N/2-i}) (in NR notation) which is the result
""   // of the complex-to-complex transform of the interleaved real data.
""   // Here the pairs are in:
""   // (r0_1,r2_2), 
""   // (r1_1,r1_2),
""   // (r2_1,r0_2) 
""   // (!!!) 
""   // each such pair will give a pair of results (F_i, F_{N/2-i}) of the final DFT
""   // of the real input data. We then compute the power and store it in the result 
""   // array (so six power values are computed per thread)
""
""   //Special handling of wrap around for the case globalId==0
"    if (globalId==0){    \n"
"        r2_2=r0_1;   \n"
"        r1_2=r2_1;   \n"
"        r0_2=r1_1;   \n"
"    }\n"
""   
"    float2 res; \n"
"    float2 tf= localTwiddle_mod0[localId];\n"
"    untangle(res,r0_1,r2_2,tf); \n"
"    if(0 == globalId) { \n"
"       res.x=res.y= 0.0f; \n"
"    } \n"
"    if(!ignore) ps_data[globalId] = norm_factor * (res.x * res.x + res.y * res.y); \n"
""
"    tf= localTwiddle_mod1[localId]; \n"
"    untangle(res,r1_1,r1_2,tf);  \n"
"    if(!ignore) ps_data[globalId+subFFTlength] = norm_factor * (res.x * res.x + res.y * res.y); \n"
""
"    tf= localTwiddle_mod2[localId];\n"
"    untangle(res,r2_1,r0_2,tf);  \n"
"    if(!ignore) ps_data[globalId+subFFTlength+subFFTlength] = norm_factor * (res.x * res.x + res.y * res.y); \n"
""
""   //TODO optimze store, avoid writing in reverse order wrt. thread index
""   // for globalId == subFFTlength /2, the following code actually repeats exactly the 
""   // same results as for the three previous array positions. 
""   // we avoid the overhead to prevent this.
""   // for globalId == 0, two indices are re-computed redundantly, plus the extra element at 
""   // index 3*subFFlength (== fullFFTlength) is taken care of, so this avoids some special
""   // case handling 
""
"    tf= localTwiddle_modM0[localId]; \n"
"    untangle(res,r0_2,r2_1,tf);  \n"
"    if(!ignore) ps_data[subFFTlength-globalId] = norm_factor * (res.x * res.x + res.y * res.y); \n"
"    tf= localTwiddle_modM1[localId]; \n"
"    untangle(res,r1_2,r1_1,tf);  \n"
"    if(!ignore) ps_data[subFFTlength+subFFTlength-globalId] = norm_factor * (res.x * res.x + res.y * res.y); \n"
"    tf= localTwiddle_modM2[localId]; \n"
"    untangle(res,r2_2,r0_1,tf);  \n"
"    if(!ignore) ps_data[subFFTlength+subFFTlength+subFFTlength-globalId] = norm_factor * (res.x * res.x + res.y * res.y); \n"
""
"  \n"
"} \n";




static const char kernelFillFloatBuffer[] =
"__kernel void kernelFillFloatBuffer(__private unsigned int  length, \n"
"                                    __private float         value, \n"
"                                    __global  float        *buffer) \n"
"{ \n"
"    size_t i = get_global_id(0); \n"
""
"    if(i < length) { \n"
"        buffer[i] = value; \n"
"    } \n"
"} \n";

static const char kernelFillIntBuffer[] =
"__kernel void kernelFillIntBuffer(__private unsigned int  length, \n"
"                                    __private int           value, \n"
"                                    __global  int          *buffer) \n"
"{ \n"
"    size_t i = get_global_id(0); \n"
""
"    if(i < length) { \n"
"        buffer[i] = value; \n"
"    } \n"
"} \n";




static const char kernelHarmonicSumming[] =
"#ifdef USE_TEXTURES \n"
"    /* FIXME: add OpenCL image object access here */ \n"
"    #define FETCH(t, i) tex1Dfetch(t##Image, i) \n"
"#else \n"
"    #define FETCH(t, i) (t[i]) \n"
"#endif\n"
""
"__kernel void kernelHarmonicSumming(__global   float       *sumspec1, \n"
"                                    __global   float       *sumspec2, \n"
"                                    __global   float       *sumspec3, \n"
"                                    __global   float       *sumspec4, \n"
"                                    __global   const float *powerspectrum, \n"
"                                    __global   int         *dirty,    \n"  
"                                    __constant int         *h_lut, \n"
"                                    __constant int         *k_lut, \n"
"                                    __constant float       *thrA, \n"
"                                    __private  int          window_2, \n"
"                                    __private  int          fundamental_idx_hi, \n"
"                                    __private  int          harmonic_idx_hi) \n"
"{ \n"
""
"    __local float sspec_cand[4 * OCL_HS_WGSIZE_X]; \n"
"    /* FIXME: these pointers should be stored locally or in registers! */ \n"
"    __global float * sumspec[4]; \n"
""
"    int idx_j = (get_group_id(1) << 4) + get_group_id(0); \n"
"    int idx_j_offset =  (idx_j << OCL_HS_LOG_WGSIZE_X) + -16 ; // negative index to handle left border \n"
"    int i = idx_j_offset + get_local_id(0) + 8; \n"
"    int k; \n"
"    int h = i; \n"
"    int j, jj,len, offset, lend2, lenM1; \n"
"    float sum; \n"
"    float p; \n"
"    int i2, i4, i8; \n"
"    int iN; \n"
""
"    if(i < window_2 || i >= harmonic_idx_hi ) {       \n"
"    // no candidate contribution from this index      \n"
"    sspec_cand[get_local_id(0)] = 0.0f;               \n"
"    sspec_cand[OCL_HS_WGSIZE_X + get_local_id(0)] = 0.0f; \n"
"    sspec_cand[2*OCL_HS_WGSIZE_X + get_local_id(0)] = 0.0f; \n"
"    sspec_cand[3*OCL_HS_WGSIZE_X + get_local_id(0)] = 0.0f; \n"
" } else {        \n"
""
"    p  = FETCH(powerspectrum, i); \n"
"    i2 = i  + i; \n"
"    i4 = i << 2; \n"
"    i8 = i4 + i4; \n"
"    iN = i8 + 8; \n"
""
"    // mark pages with potential candidates also in powerspectrum itself  \n"
"    // this can cause concurrent writes to the same address, but all of them  \n" 
"    // write the same value !                                             \n"
"    if( (p > FETCH(thrA,0)) && (i < fundamental_idx_hi)) {                \n"
"        dirty[ (i>> OCL_LOG_PS_PAGE_SIZE_X)] = 1;                               \n"
"    }                                                                     \n"
""
"    p += FETCH(powerspectrum, iN >> 4); \n"
"    sspec_cand[get_local_id(0)] = p; \n"
""
"    iN   = i4 + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i8; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"

"    p += sum; \n"
"    sspec_cand[OCL_HS_WGSIZE_X + get_local_id(0)] = p; \n"
""
"    iN   = i2 + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
""
"    p += sum; \n"
"    sspec_cand[2 * OCL_HS_WGSIZE_X + get_local_id(0)] = p; \n"
""
"    iN   = i + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
""
"    p += sum; \n"
"    sspec_cand[3 * OCL_HS_WGSIZE_X + get_local_id(0)] = p; \n"
" } \n"
""
"    sumspec[0] = sumspec1; \n"
"    sumspec[1] = sumspec2; \n"
"    sumspec[2] = sumspec3; \n"
"    sumspec[3] = sumspec4; \n"
""
"    barrier(CLK_GLOBAL_MEM_FENCE); \n"
""
"    if((get_local_id(0) & 15) < 12) { \n"
""
"        h = FETCH(h_lut, get_local_id(0) & 15); \n"
"        k = FETCH(k_lut, get_local_id(0) & 15) + ((get_local_id(0) >> 4) << 4); \n"
""
"        len    = 1 << h; \n"
"        lend2  = len >> 1; \n"
"        offset = ((h - 1) << OCL_HS_LOG_WGSIZE_X) + k; \n"
"        lenM1  = len - 1; \n"
""
"        sum = sspec_cand[offset]; \n"
""
"        /* #pragma unroll 15 */ \n"
"        for (j = 1; j < 16; j++) { \n"
"            sum = fmax(sum, sspec_cand[offset + (j & lenM1)]); \n"
"        } \n"
""
"        jj = (idx_j_offset + k + 8 + lend2) ; \n"
"        j= (jj>= 0) ? (jj  >> h) : -1 ;       \n"
""
"        if ( \n"
"            (sum > FETCH(thrA, h )) && \n"
"            (j < fundamental_idx_hi) && j>=0 ) \n"
"        { \n"
"        /* mark this page of the sumspec array as dirty */                      \n"       
"        /* TODO: maybe write to shared memory buffer first, then (by thread 0) to global memory if set to 1, avoiding \n"
"                simultaneous writes to global memory from the same block */                                          \n"
"\n"
"            dirty[((fundamental_idx_hi >> OCL_LOG_PS_PAGE_SIZE_X) +1)  * h + (j>> OCL_LOG_PS_PAGE_SIZE_X)] = 1;                  \n"
"            sumspec[h-1][j] = sum; \n"   
"        } \n"
"    } \n"
"} \n";


static const char kernelHarmonicSummingGaps[] =
"__kernel void kernelHarmonicSummingGaps(__global   float       *sumspec1, \n"
"                                        __global   float       *sumspec2, \n"
"                                        __global   float       *sumspec3, \n"
"                                        __global   float       *sumspec4, \n"
"                                        __global   const float *powerspectrum, \n"
"                                        __global   int         *dirty,    \n"
"                                        __constant int         *h_lut, \n"
"                                        __constant int         *k_lut, \n"
"                                        __constant float       *thrA, \n"
"                                        __private  int          window_2, \n"
"                                        __private  int          fundamental_idx_hi, \n"
"                                        __private  int          harmonic_idx_hi) \n"
"{ \n"
""
"    __local float sspec_cand[2 * OCL_HS_WGSIZE_X]; \n"
"    /* FIXME: these pointers should be stored locally or in registers! */ \n"
"    __global float * sumspec[4]; \n"
""
"    int idx_j = (get_group_id(1) << 4) + get_group_id(0); \n"
"    int idx_j_offset = (idx_j << OCL_HS_LOG_WGSIZE_X); \n"
"    int idx_i_offset = get_local_id(0); \n"
"    int i = idx_j_offset + 4 + (get_local_id(0) & 7) + ((get_local_id(0) >> 3 ) << 4); \n"
"    int k; \n"
"    int h = i; \n"
"    int j, len, offset, lend2, lenM1; \n"
"    float sum; \n"
"    float p; \n"
"    int i2, i4, i8; \n"
"    int iN; \n"
"    /* for this kernel , there can be now overlap with the left border of the spectrum (index 0), \n"
"     * but we could be lower than window_2 or higher than harmonic_idx_hi */\n"
"    if(i < window_2 || i >= harmonic_idx_hi ) {                            \n"
"      // no candidate contribution from this index                         \n"
"       sspec_cand[idx_i_offset] = 0.0f;                                    \n"
"       sspec_cand[(OCL_HS_WGSIZE_X / 2)  + idx_i_offset] = 0.0f;           \n"
"       sspec_cand[2*(OCL_HS_WGSIZE_X / 2) + idx_i_offset] = 0.0f;          \n"
"       sspec_cand[3*(OCL_HS_WGSIZE_X / 2) + idx_i_offset] = 0.0f;          \n"
"   } else {                                                                \n"
""
"    p  = FETCH(powerspectrum, i); \n"
"    i2 = i + i; \n"
"    i4 = i << 2; \n"
"    i8 = i4 + i4; \n"
"    iN = i8 + 8; \n"
"    // mark pages with potential candidates also in powerspectrum itself  \n"
"    // this can cause concurrent writes to the same address, but all of them  \n"
"    // write the same value !                                             \n"
"    if( (p > FETCH(thrA,0)) && (i < fundamental_idx_hi) ) {                \n"
"        dirty[ (i>> OCL_LOG_PS_PAGE_SIZE_X)] = 1;                               \n"
"    }  \n"
""
"    p += FETCH(powerspectrum, iN >> 4); \n"
"    sspec_cand[idx_i_offset] = p; \n"
""
"    iN   = i4 + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i8; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
""
"    p += sum; \n"
"    sspec_cand[(OCL_HS_WGSIZE_X / 2) + idx_i_offset] = p; \n"
""
"    iN   = i2 + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i4; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
""
"    p += sum; \n"
"    sspec_cand[2 * (OCL_HS_WGSIZE_X / 2) + idx_i_offset] = p; \n"
""
"    iN   = i + 8; \n"
"    sum  = FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
"    iN  += i2; \n"
"    sum += FETCH(powerspectrum, iN >> 4); \n"
""
"    p += sum; \n"
"    sspec_cand[3 * (OCL_HS_WGSIZE_X / 2) + idx_i_offset] = p; \n"
""
"    sumspec[0] = sumspec1; \n"
"    sumspec[1] = sumspec2; \n"
"    sumspec[2] = sumspec3; \n"
"    sumspec[3] = sumspec4; \n"
"  } \n"
"    barrier(CLK_GLOBAL_MEM_FENCE); \n"
""
"    if((get_local_id(0) < 4 * (OCL_HS_WGSIZE_X / 16)) && ((get_local_id(0) & 3) != 3)) { \n"
"        h      = 3 - (get_local_id(0) & 3); \n"
"        k      = FETCH(k_lut, (get_local_id(0) & 3) + 13) + ((get_local_id(0) >> 2) << 3); \n"
"        len    = 1 << h; \n"
"        lend2  = len >> 1; \n"
"        offset = ((h - 1) << (OCL_HS_LOG_WGSIZE_X - 1)) + k; \n"
"        lenM1  = len - 1; \n"
""
"        sum = sspec_cand[offset]; \n"
""
"        /* #pragma unroll 7 */ \n"
"        for (j = 1; j < 8; j++) { \n"
"            sum = fmax(sum, sspec_cand[offset + (j & lenM1)]); \n"
"        } \n"
""
"        j = ((idx_j_offset + k + ((get_local_id(0) >> 2) << 3) + 4 + lend2) >> h) ; \n"
""
"        if ( \n"
"            (sum > FETCH(thrA, h )) && \n"
"            (j < fundamental_idx_hi)) \n"
"        { \n"
"        /* mark this page of the sumspec array as dirty */                      \n"
"        /* TODO: maybe write to shared memory buffer first, then (by thread 0) to global memory if set to 1, avoiding \n"
"                simultaneous writes to global memory from the same block */                                          \n"
"\n"
"            dirty[((fundamental_idx_hi >> OCL_LOG_PS_PAGE_SIZE_X) +1)  * h + (j>> OCL_LOG_PS_PAGE_SIZE_X)] = 1;                  \n"
"            sumspec[h-1][j] = sum; \n"
"        } \n"
"    } \n"
"} \n";


#endif
