UPGRADE YOUR BROWSER

We have detected your current browser version is not the latest one. Xilinx.com uses the latest web technologies to bring you the best online experience possible. Please upgrade to a Xilinx.com supported browser:Chrome, Firefox, Internet Explorer 11, Safari. Thank you!

cancel
Showing results for 
Search instead for 
Did you mean: 
Highlighted
Observer lzhao
Observer
133 Views
Registered: ‎07-17-2019

decent pickle error

 

I am running decent tool, and I got following error, which seems related with pickle. But if I am testing my input function stand alone, it won't give this error message. I don't know if it's dnndk tool relaated or not. Please advise. Thanks!

#####################################
QUANTIZE
#####################################
Using TensorFlow backend.
INFO: Checking Float Graph...
Traceback (most recent call last):
  File "/harddisk1/xxxx/anaconda2/envs/decent/bin/decent_q", line 10, in <module>
    sys.exit(run_main())
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/contrib/decent_q/python/decent_q.py", line 540, in run_main
    app.run(main=my_main, argv=[sys.argv[0]] + unparsed)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 125, in run
    _sys.exit(main(argv))
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/contrib/decent_q/python/decent_q.py", line 539, in <lambda>
    my_main = lambda unused_args: main(unused_args, FLAGS)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/contrib/decent_q/python/decent_q.py", line 335, in main
    quantize_frozen(input_graph_def, input_fn, q_config, s_config, flags.skip_check)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/contrib/decent_q/python/decent_q.py", line 222, in quantize_frozen
    check_float_graph(input_graph_def, input_fn, q_config, s_config)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/site-packages/tensorflow/contrib/decent_q/python/decent_q.py", line 143, in check_float_graph
    inputs = input_fn(iter=0)
  File "./unet_input_fn.py", line 45, in calib_input
    data = pickle.load(f)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/pickle.py", line 1384, in load
    return Unpickler(file).load()
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/pickle.py", line 864, in load
    dispatch[key](self)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/pickle.py", line 1075, in load_inst
    klass = self.find_class(module, name)
  File "/harddisk1/xxxx/anaconda2/envs/decent/lib/python2.7/pickle.py", line 1132, in find_class
    klass = getattr(mod, name)
AttributeError: 'module' object has no attribute 'Data'
#####################################
QUANTIZATION COMPLETED
#####################################

My decent shell script:

decent_q quantize \
  --input_frozen_graph ./noise_filter_model/Unet.pb \
  --input_nodes input_1 \
  --input_shapes ?,1024,1024,6 \
  --output_nodes conv2d_23/Sigmoid \
  --method 1 \
  --input_fn some_input_fn.calib_input

The input function:

import pickle
#from parseHit import *
#from plotLayer import *
import glob
import time
import numpy as np
from scipy.ndimage import zoom
from skimage.measure import block_reduce
from scipy import sparse as sp
import keras
from keras.backend.tensorflow_backend import set_session
#from preprocess import *
from data import Data
#import tensorflow
#from tensorflow.keras.backend import set_session


width = 1024
data_shape=(width, width, 6)
batch_size = 1
in_dir = "smallDataset"

path = in_dir
cached_data = {}
file_list = glob.glob(in_dir + "/*")
ratio = 1024 / data_shape[0]
#data_shape = data_shape
output = 'image'
batch_in = []
batch_out = []


def calib_input(iter):
    np.random.shuffle(file_list)
    batch_in = np.zeros((batch_size,) + data_shape)

    if output == 'image':
        batch_out = np.zeros((batch_size,) + data_shape)
    elif output == 'label':
        batch_out = np.zeros(batch_size)

    for i in range(len(file_list)):
        with open(file_list[i], "rb") as f:
            data = pickle.load(f)
            #data = []
            max_reduce(data)
            cached_data[file_list[i]] = data

    #while True:
    for i in range(batch_size):
        f_ind = np.random.randint(0, len(file_list))
            #if os.path.basename(self.file_list[f_ind]) in self.cached_data:
            #    data = self.cached_data[self.file_list[f_ind]]
            #else:
            #    with open(self.file_list[f_ind], "rb") as f:
            #        data = pickle.load(f)
            #        self.max_reduce(data)
            #        self.cached_data[self.file_list[f_ind]] = data
        data = cached_data[file_list[f_ind]]
        ind = np.random.randint(0, len(data))

            #train_data = map(lambda x: block_reduce(x.toarray(), block_size=(self.ratio,self.ratio), func=np.max), data[ind].hL)
        train_data = data[ind].train
            #train_data = np.swapaxes(train_data, 0, 2)
        batch_in[i] = np.heaviside(train_data, 0)

        if output == 'image':
                #gt_data = map(lambda x: block_reduce(x.toarray(), block_size=(self.ratio,self.ratio), func=np.max), data[ind].gthL)
            gt_data = data[ind].image
                #gt_data = np.swapaxes(gt_data, 0, 2)
        elif output == 'label':
            gt_data = np.array(data[ind].label)
        batch_out[i] = np.heaviside(gt_data, 0)

    #yield batch_in, batch_out
    #print(batch_in)
    return {"input_1": batch_in}
0 Kudos
1 Reply
Xilinx Employee
Xilinx Employee
102 Views
Registered: ‎03-27-2013

Re: decent pickle error

Hi @lzhao ,

 

I agree with you that it is more like a pickle issue than a DNNDK issue here.

I would suggest you to search on internet. You may find some useful posts.

Here is my two cents:

https://www.stefaanlippens.net/python-pickling-and-dealing-with-attributeerror-module-object-has-no-attribute-thing.html

 

Best Regards,
Jason
-----------------------------------------------------------------------------------------------
Please mark the Answer as "Accept as solution" if the information provided is helpful.

Give Kudos to a post which you think is helpful and reply oriented.
-----------------------------------------------------------------------------------------------