xf::equalizeHist does not work with pragma dataflow

[eewse]

The following prgram can run on vivado for

TEST001 - TEST004

but for

TEST005

there will produce no output. But why?

#include "axi_sgbm_config.h"
#include "xf_config_params.h"
#include "imgproc/xf_hist_equalize.hpp"
#include "imgproc/xf_duplicateimage.hpp"
#include "imgproc/xf_add_weighted.hpp"
#include "imgproc/xf_convert_bitdepth.hpp"
#include "core/xf_arithm.hpp"

#define TEST005

template <typename T, int rows, int cols>
void drain_xf_mat(T& img) {
    xf::Scalar<1, ap_uint<24>> p;
    for(int i=0; i < rows*cols; i++)
        #pragma HLS pipeline rewind
        //#pragma HLS loop_flatten
        p = img.data[i];
}

void equalizer(XGRAY_DST_IMAGE& src_disp, XGRAY_DST_IMAGE& src_disp4) {

#pragma HLS dataflow
#pragma HLS inline
    XGRAY_DST_IMAGE src_disp2(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp3(dst_rows, dst_cols);
    #pragma HLS stream variable=src_disp2 dim=1 depth=128
    #pragma HLS stream variable=src_disp3 dim=1 depth=128

    xf::duplicateMat<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp, src_disp2, src_disp3);
    xf::equalizeHist<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp2, src_disp3, src_disp4);

}


void sgbm_filter_simple(AXI_STREAM& src_data_l, AXI_STREAM& src_data_r, AXI_STREAM& src_data_d)
{
#pragma HLS INTERFACE axis port=src_data_l
#pragma HLS INTERFACE axis port=src_data_r
#pragma HLS INTERFACE axis port=src_data_d

#pragma HLS stream variable=src_data_l dim=1 depth=64
#pragma HLS stream variable=src_data_r dim=1 depth=64
#pragma HLS stream variable=src_data_d dim=1 depth=64

XGRAY_SRC_IMAGE src_l(src_rows, src_cols);
XGRAY_SRC_IMAGE src_r(src_rows, src_cols);
#pragma HLS stream variable=src_l dim=1 depth=128

#ifdef TEST001
    #pragma HLS dataflow
    xf::AXIvideo2xfMat(src_data_l, src_l);
    xf::AXIvideo2xfMat(src_data_r, src_r);
    drain_xf_mat<XGRAY_SRC_IMAGE, src_rows, src_cols>(src_r);
    xf::xfMat2AXIvideo(src_l, src_data_d);
#elif defined(TEST002)
    #pragma HLS stream variable=src_r dim=1 depth=128
    XGRAY_DST_IMAGE src_disp(src_rows, src_cols);
    #pragma HLS stream variable=src_disp dim=1 depth=128
    #pragma HLS dataflow
    xf::AXIvideo2xfMat(src_data_l, src_l);
    xf::AXIvideo2xfMat(src_data_r, src_r);
    xf::bitwise_and<XF_8UC1, src_rows, src_cols, XF_NPPC1> (src_l, src_r, src_disp);
    xf::xfMat2AXIvideo(src_disp, src_data_d);
#elif defined(TEST003)
    #pragma HLS stream variable=src_r dim=1 depth=128
    XGRAY16_DST_IMAGE src_disp16(src_rows, src_cols);
    XGRAY_DST_IMAGE src_disp(src_rows, src_cols);
    #pragma HLS stream variable=src_disp16 dim=1 depth=128
    #pragma HLS stream variable=src_disp dim=1 depth=128
    #pragma HLS dataflow
    xf::AXIvideo2xfMat(src_data_l, src_l);
    xf::AXIvideo2xfMat(src_data_r, src_r);
    xf::addWeighted<XF_8UC1, XF_16UC1, src_rows, src_cols, XF_NPPC1> (src_l, 0.5, src_r, 0.5, 0., src_disp16);
    xf::convertTo<XF_16UC1, XF_8UC1, src_rows, src_cols, XF_NPPC1> (src_disp16, src_disp, XF_CONVERT_16U_TO_8U, 0);
    xf::xfMat2AXIvideo(src_disp, src_data_d);
#elif defined(TEST004)
    #pragma HLS stream variable=src_r dim=1 depth=128
    #pragma HLS dataflow
    XGRAY16_DST_IMAGE src_disp16(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp4(dst_rows, dst_cols);
    #pragma HLS stream variable=src_disp16 dim=1 depth=128
    #pragma HLS stream variable=src_disp dim=1 depth=128
    #pragma HLS stream variable=src_disp4 dim=1 depth=1024
    XGRAY_DST_IMAGE src_disp2(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp3(dst_rows, dst_cols);
    #pragma HLS stream variable=src_disp2 dim=1 depth=1024
    #pragma HLS stream variable=src_disp3 dim=1 depth=1024

    xf::AXIvideo2xfMat(src_data_l, src_l);
    xf::AXIvideo2xfMat(src_data_r, src_r);
    xf::addWeighted<XF_8UC1, XF_16UC1, src_rows, src_cols, XF_NPPC1> (src_l, 0.5, src_r, 0.5, 0., src_disp16);
    xf::convertTo<XF_16UC1, XF_8UC1, src_rows, src_cols, XF_NPPC1> (src_disp16, src_disp, XF_CONVERT_16U_TO_8U, 0);
    xf::duplicateMat<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp, src_disp2, src_disp3);
    xf::equalizeHist<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp2, src_disp3, src_disp4);
    //xf::bitwise_and<XF_8UC1, src_rows, src_cols, XF_NPPC1> (src_disp2, src_disp3, src_disp4);
    xf::xfMat2AXIvideo(src_disp4, src_data_d);
#elif defined(TEST005)
      #pragma HLS stream variable=src_r dim=1 depth=128
    #pragma HLS dataflow
    XGRAY16_DST_IMAGE src_disp16(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp4(dst_rows, dst_cols);
    #pragma HLS stream variable=src_disp16 dim=1 depth=128
    #pragma HLS stream variable=src_disp dim=1 depth=128
    #pragma HLS stream variable=src_disp4 dim=1 depth=1024
    XGRAY_DST_IMAGE src_disp2(dst_rows, dst_cols);
    XGRAY_DST_IMAGE src_disp3(dst_rows, dst_cols);
    #pragma HLS stream variable=src_disp2 dim=1 depth=1024
    #pragma HLS stream variable=src_disp3 dim=1 depth=1024

    xf::AXIvideo2xfMat(src_data_l, src_l);
    xf::AXIvideo2xfMat(src_data_r, src_r);
    xf::addWeighted<XF_8UC1, XF_16UC1, src_rows, src_cols, XF_NPPC1> (src_l, 0.5, src_r, 0.5, 0., src_disp16);
    xf::convertTo<XF_16UC1, XF_8UC1, src_rows, src_cols, XF_NPPC1> (src_disp16, src_disp, XF_CONVERT_16U_TO_8U, 0);
    xf::duplicateMat<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp, src_disp2, src_disp3);
    xf::equalizeHist<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src_disp2, src_disp3, src_disp4);
    xf::xfMat2AXIvideo(src_disp4, src_data_d); 
#endif
}

[eewse]

No need to be too demanding. It won't work even it is only few lines

#include "top.h"
#include "imgproc/xf_hist_equalize.hpp"
#include "imgproc/xf_duplicateimage.hpp"

void equalizeHist(AXI_STREAM& src_axi, AXI_STREAM& dst_axi)
{

#pragma HLS INTERFACE axis port=src_axi
#pragma HLS INTERFACE axis port=dst_axi

#pragma HLS dataflow
XGRAY_SRC_IMAGE  src(src_rows, src_cols);
XGRAY_DST_IMAGE  dst(dst_rows, dst_cols);
XGRAY_DST_IMAGE  dst1(dst_rows, dst_cols);
XGRAY_DST_IMAGE  dst2(dst_rows, dst_cols);
#pragma HLS STREAM variable=src depth=64
#pragma HLS STREAM variable=dst depth=64
#pragma HLS STREAM variable=dst1 depth=64
#pragma HLS STREAM variable=dst2 depth=64

    xf::AXIvideo2xfMat(src_axi, src);
    xf::duplicateMat<XF_8UC1, src_rows, src_cols, XF_NPPC1>(src, dst1, dst2);
    xf::equalizeHist<XF_8UC1, dst_rows, dst_cols, XF_NPPC1>(dst1, dst2, dst);
    xf::xfMat2AXIvideo(dst, dst_axi);
}

[mohammadhgh]

Hi @eewse ,

I have the same problem with xf::equalizeHist in dataflow region. Could you please share your solution if you have already solved it?

[eewse]

I don't have any solution.

[mohammadhgh · Accepted solution]

I found a way to do histogram equalization in dataflow region which works for my application. I explain it here maybe it will be usefull for others.

xf::equalizeHist has two processes which cannot be done concurrently inside a dataflow region. The first one (xFHistogramKernel) computes a histogram and the second one (xFEqualize) equalizes the image according to the computed histogram. What I observed in the chipscope was that while streaming the data into two input ports of xf::equalizeHist function, the second input port doesn't accept data until the data transfer to the first input is finished. 

In my application, the video frames don't change rapidly, so I can use the histogram values computed for the last frame to equalize the current frame. I did this by defining a static array for histogram values. Also I was not able to use xf::duplicateMat function, so I defined two input streaming ports to the toplevel function and used an axis_broadcaster IP in the Vivado ip-integrator.

The source code of my application is this:

#include "xf_median_blur_config.h"

void filters_accel(hls::stream< ap_axiu<8,1,1,1> >& _src1,
	           hls::stream< ap_axiu<8,1,1,1> >& _src2,
	           hls::stream< ap_axiu<8,1,1,1> >& _dst,
		   uint32_t histogram1 [256],
	           uint32_t histogram2 [1][256])

{
#pragma HLS INTERFACE axis register both  port=_src1
#pragma HLS INTERFACE axis register both  port=_src2
#pragma HLS INTERFACE axis register both  port=_dst

// No control interface - auto-start as soon as there's an input frame
#pragma HLS INTERFACE ap_ctrl_none port=return  // no handshakes

	 xf::Mat<TYPE, HEIGHT, WIDTH, NPxPC> imgInput1(HEIGHT,WIDTH);
	 xf::Mat<TYPE, HEIGHT, WIDTH, NPxPC> imgInput2(HEIGHT,WIDTH);
	 xf::Mat<TYPE, HEIGHT, WIDTH, NPxPC> imgOutput(HEIGHT,WIDTH);

#pragma HLS stream variable=imgInput1.data depth=1337
#pragma HLS stream variable=imgInput2.data depth=1337
#pragma HLS stream variable=imgOutput.data depth=1337

	#pragma HLS dataflow

	xf::AXIvideo2xfMat(_src1, imgInput1);
	xf::AXIvideo2xfMat(_src2, imgInput2);

	xf::calcHist<TYPE, HEIGHT, WIDTH, NPxPC>(imgInput1, histogram1);
	xf::xFEqualize<TYPE, HEIGHT, WIDTH, XF_DEPTH(TYPE,NPxPC), NPxPC, XF_WORDWIDTH(TYPE,NPxPC), (WIDTH >> XF_BITSHIFT(NPxPC))> (imgInput2, histogram2,imgOutput, 1337, 1337);

	xf::xfMat2AXIvideo(imgOutput, _dst);

}

void filters_accel_ip(hls::stream< ap_axiu<8,1,1,1> >& _src1,
		      hls::stream< ap_axiu<8,1,1,1> >& _src2,
		      hls::stream< ap_axiu<8,1,1,1> >& _dst)
{
#pragma HLS INTERFACE axis register both  port=_src1
#pragma HLS INTERFACE axis register both  port=_src2
#pragma HLS INTERFACE axis register both  port=_dst

// No control interface - auto-start as soon as there's an input frame
#pragma HLS INTERFACE ap_ctrl_none port=return  // no handshakes

	unsigned int histogram1 [256];
	static unsigned int histogram2 [1][256] = {0};

	filters_accel(_src1, _src2, _dst, histogram1, histogram2);

	for(int i=0; i<256; i++)
	{
		histogram2 [0][i] = histogram1 [i];
	}

}

[eewse]

@mohammadhgh, 

Thank you for your script.

Do you need to duplicate your last frame in your vivado project?

[mohammadhgh]

@eewse 

No, last frame is processed correctly with the histogram values calculated from the previous frame. The problem is with the first frame for which calculated histogram is available. However, in my application the system has a setup time and this can easily be ignored.

[eewse]

I am confused.  EqualizeHist should only need one image to process.  Why can't we simply duplicate the image?

Below is a typical example for opencv in C++,

#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>

using namespace cv;
using namespace std;

int main( int argc, char** argv )
{
CommandLineParser parser( argc, argv, "{@input | lena.jpg | input image}" );
Mat src=imread( samples::findFile( parser.get<String>( "@input" ) ), IMREAD_COLOR );
if( src.empty() )
{
cout << "Could not open or find the image!\n" << endl;
cout << "Usage: " << argv[0] << " <Input image>" << endl;
return -1;
}

cvtColor( src, src, COLOR_BGR2GRAY );

Mat dst;
equalizeHist( src, dst );

imshow( "Source image", src );
imshow( "Equalized Image", dst );

waitKey();

return 0;

}