Hi,
Thank you very much for the workaround, and I will indeed open an issue for this nestedarray
functionality.
This is getting a bit frustrating. Now the problem seems to be the np.concatenate
function. I tried to convert the ndarray
argument int(baseline) - waveform_data[rr_old_short['channel'] == ch][rr_start:records_required]
into a tuple
(before passing it to the np.concatenate
), but then Numba has also some problem with the function tuple()
of the conversion. I also tried to convert it into a list
but it does not work (using the tolist()
method).
This is the code I am using now (similar to that of before) but now it gives a TypeError
related to the np.concatenate
:
import numpy as np
import numba
import math
# Channels
active_channel_small = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.int16)
# Data
rec_type = np.dtype([
(('Start time', 'time'), '<i8'),
(('Length of interval ', 'length'), '<i4'),
(('Width ', 'dt'), '<i2'),
(('Channel', 'channel'), '<i2'),
(('Length of pulse ', 'pulse_length'), '<i4'),
(('Fragment', 'record_i'), '<i2'),
(('Baseline', 'baseline'), '<i2'),
(('Waveform data', 'data'), '<i2', (500,))
])
rr_old_sample = np.array(
[
(1596395008015038110, 500, 10, 0, 98304, 0, 0, [13812, 13813, 13813, 13814, 13812, 13815, 13809, 13815, 13812, 13811, 13810, 13815, 13812, 13815, 13812, 13812, 13812, 13811, 13808, 13811, 13812, 13813, 13813, 13813, 13811, 13813, 13812, 13814, 13811, 13811, 13809, 13811, 13810, 13812, 13809, 13815, 13813, 13813, 13810, 13810, 13811, 13814, 13812, 13812, 13809, 13811, 13809, 13812, 13811, 13812, 13810, 13814, 13813, 13815, 13810, 13814, 13813, 13811, 13813, 13813, 13811, 13812, 13812, 13812, 13812, 13813, 13814, 13813, 13811, 13811, 13809, 13811, 13808, 13811, 13810, 13813, 13809, 13811, 13809, 13812, 13813, 13817, 13811, 13815, 13812, 13815, 13809, 13813, 13810, 13813, 13811, 13811, 13810, 13811, 13810, 13812, 13810, 13813, 13810, 13813, 13811, 13812, 13811, 13815, 13814, 13814, 13810, 13814, 13814, 13816, 13811, 13816, 13813, 13814, 13816, 13816, 13813, 13815, 13813, 13809, 13810, 13812, 13813, 13814, 13813, 13811, 13809, 13812, 13814, 13811, 13811, 13816, 13811, 13813, 13809, 13808, 13809, 13811, 13810, 13811, 13809, 13809, 13808, 13814, 13810, 13812, 13813, 13813, 13813, 13814, 13812, 13816, 13812, 13816, 13811, 13814, 13812, 13813, 13815, 13814, 13812, 13814, 13812, 13814, 13813, 13817, 13814, 13813, 13811, 13812, 13810, 13813, 13809, 13815, 13811, 13813, 13811, 13812, 13810, 13811, 13812, 13811, 13809, 13813, 13809, 13811, 13810, 13811, 13811, 13812, 13811, 13815, 13809, 13815, 13814, 13811, 13811, 13814, 13811, 13811, 13813, 13814, 13810, 13813, 13809, 13814, 13810, 13810, 13811, 13812, 13809, 13813, 13809, 13812, 13811, 13812, 13812, 13810, 13809, 13811, 13813, 13814, 13813, 13814, 13812, 13814, 13810, 13813, 13813, 13813, 13813, 13814, 13810, 13813, 13811, 13814, 13811, 13814, 13811, 13815, 13813, 13813, 13811, 13813, 13812, 13815, 13811, 13813, 13812, 13811, 13809, 13812, 13811, 13812, 13811, 13815, 13809, 13813, 13811, 13811, 13810, 13814, 13809, 13812, 13809, 13811, 13810, 13812, 13809, 13813, 13812, 13810, 13810, 13812, 13812, 13812, 13812, 13814, 13811, 13814, 13813, 13812, 13810, 13813, 13812, 13814, 13812, 13812, 13813, 13816, 13811, 13812, 13813, 13817, 13811, 13812, 13812, 13813, 13813, 13816, 13814, 13814, 13813, 13814, 13811, 13813, 13812, 13814, 13813, 13814, 13812, 13813, 13812, 13812, 13812, 13811, 13809, 13813, 13811, 13812, 13810, 13810, 13810, 13813, 13813, 13812, 13810, 13810, 13814, 13812, 13810, 13814, 13811, 13813, 13811, 13812, 13809, 13813, 13809, 13812, 13812, 13811, 13811, 13814, 13810, 13812, 13811, 13814, 13813, 13816, 13811, 13812, 13808, 13814, 13810, 13815, 13810, 13812, 13809, 13813, 13811, 13812, 13810, 13814, 13810, 13812, 13812, 13812, 13809, 13812, 13812, 13813, 13812, 13812, 13810, 13811, 13809, 13813, 13812, 13813, 13811, 13815, 13811, 13813, 13811, 13815, 13811, 13811, 13810, 13813, 13810, 13811, 13810, 13814, 13809, 13811, 13810, 13814, 13810, 13813, 13813, 13812, 13808, 13813, 13811, 13812, 13811, 13812, 13813, 13815, 13812, 13813, 13812, 13814, 13809, 13813, 13807, 13810, 13808, 13811, 13812, 13813, 13811, 13813, 13812, 13814, 13809, 13812, 13814, 13814, 13810, 13813, 13808, 13812, 13810, 13813, 13814, 13813, 13811, 13812, 13810, 13812, 13812, 13814, 13812, 13814, 13810, 13810, 13810, 13812, 13810, 13812, 13810, 13814, 13809, 13814, 13813, 13813, 13811, 13811, 13810, 13811, 13811, 13814, 13813, 13813, 13810, 13813, 13813, 13814, 13811, 13811, 13811, 13813, 13809, 13810, 13812, 13812, 13811, 13811, 13810, 13814, 13812, 13813, 13813, 13811, 13809, 13812, 13810, 13814, 13814, 13815, 13810, 13813, 13811, 13813, 13811, 13811, 13812, 13812]),
(1596395008015038110, 500, 10, 1, 98304, 0, 0, [13621, 13622, 13621, 13621, 13625, 13620, 13625, 13624, 13623, 13624, 13624, 13626, 13624, 13623, 13624, 13622, 13625, 13622, 13624, 13623, 13622, 13623, 13623, 13621, 13624, 13622, 13624, 13621, 13622, 13624, 13623, 13623, 13624, 13623, 13626, 13623, 13625, 13623, 13625, 13624, 13625, 13621, 13622, 13624, 13621, 13622, 13622, 13621, 13623, 13621, 13624, 13621, 13627, 13626, 13626, 13624, 13626, 13624, 13623, 13622, 13622, 13624, 13622, 13621, 13622, 13621, 13623, 13622, 13625, 13623, 13623, 13620, 13623, 13623, 13623, 13619, 13623, 13623, 13623, 13622, 13623, 13621, 13623, 13622, 13624, 13622, 13623, 13623, 13621, 13623, 13620, 13620, 13623, 13621, 13622, 13622, 13623, 13625, 13622, 13622, 13624, 13623, 13622, 13623, 13627, 13622, 13623, 13624, 13625, 13623, 13627, 13623, 13624, 13623, 13627, 13624, 13622, 13622, 13623, 13621, 13622, 13621, 13623, 13623, 13623, 13623, 13621, 13621, 13623, 13624, 13624, 13622, 13625, 13624, 13625, 13622, 13627, 13622, 13622, 13622, 13622, 13621, 13624, 13622, 13622, 13620, 13620, 13622, 13623, 13621, 13622, 13621, 13622, 13623, 13623, 13624, 13625, 13622, 13623, 13621, 13622, 13625, 13624, 13623, 13624, 13624, 13625, 13624, 13625, 13622, 13623, 13621, 13620, 13621, 13621, 13623, 13624, 13622, 13624, 13624, 13623, 13621, 13623, 13623, 13624, 13623, 13623, 13622, 13625, 13623, 13623, 13621, 13621, 13623, 13623, 13621, 13623, 13624, 13622, 13619, 13621, 13622, 13624, 13623, 13624, 13623, 13622, 13625, 13626, 13621, 13624, 13623, 13624, 13623, 13624, 13622, 13620, 13620, 13623, 13618, 13622, 13623, 13623, 13621, 13621, 13620, 13625, 13623, 13624, 13624, 13620, 13622, 13622, 13621, 13621, 13622, 13623, 13623, 13621, 13619, 13622, 13620, 13622, 13622, 13623, 13623, 13624, 13623, 13625, 13620, 13622, 13622, 13623, 13621, 13624, 13624, 13622, 13622, 13622, 13621, 13619, 13621, 13622, 13622, 13620, 13622, 13621, 13618, 13622, 13623, 13621, 13624, 13623, 13620, 13621, 13622, 13623, 13623, 13624, 13625, 13625, 13624, 13621, 13621, 13623, 13621, 13622, 13619, 13623, 13625, 13624, 13621, 13622, 13624, 13624, 13622, 13625, 13624, 13622, 13623, 13623, 13625, 13625, 13623, 13624, 13623, 13623, 13624, 13623, 13622, 13622, 13621, 13620, 13620, 13621, 13620, 13621, 13622, 13622, 13624, 13621, 13621, 13624, 13623, 13621, 13620, 13622, 13620, 13626, 13622, 13622, 13624, 13625, 13625, 13624, 13620, 13621, 13621, 13624, 13620, 13619, 13625, 13621, 13621, 13624, 13621, 13623, 13622, 13623, 13625, 13624, 13624, 13624, 13624, 13624, 13623, 13624, 13622, 13624, 13620, 13625, 13624, 13624, 13621, 13621, 13621, 13623, 13621, 13622, 13620, 13623, 13624, 13624, 13623, 13624, 13622, 13624, 13622, 13624, 13623, 13622, 13623, 13626, 13622, 13623, 13622, 13622, 13622, 13621, 13623, 13623, 13621, 13623, 13621, 13621, 13624, 13624, 13624, 13624, 13622, 13625, 13624, 13627, 13623, 13624, 13622, 13623, 13623, 13623, 13623, 13623, 13624, 13624, 13621, 13624, 13622, 13625, 13623, 13625, 13622, 13622, 13622, 13624, 13623, 13625, 13622, 13623, 13625, 13625, 13621, 13623, 13622, 13623, 13621, 13620, 13622, 13621, 13624, 13623, 13622, 13623, 13623, 13624, 13621, 13625, 13622, 13625, 13625, 13625, 13622, 13622, 13622, 13624, 13625, 13622, 13619, 13625, 13623, 13622, 13622, 13622, 13621, 13625, 13622, 13621, 13623, 13624, 13622, 13621, 13622, 13623, 13622, 13624, 13622, 13625, 13620, 13624, 13622, 13624, 13622, 13623, 13621, 13621, 13621, 13619, 13622, 13623, 13624, 13624, 13621, 13621, 13621, 13623, 13624, 13625, 13623, 13623, 13620, 13622, 13622]),
(1596395008015038110, 500, 10, 2, 98304, 0, 0, [13608, 13609, 13606, 13611, 13609, 13608, 13606, 13608, 13607, 13609, 13606, 13605, 13608, 13606, 13606, 13607, 13603, 13606, 13607, 13607, 13608, 13607, 13607, 13610, 13607, 13609, 13606, 13607, 13606, 13607, 13605, 13606, 13603, 13604, 13606, 13608, 13603, 13607, 13605, 13607, 13603, 13608, 13607, 13606, 13607, 13608, 13608, 13606, 13608, 13609, 13610, 13609, 13606, 13607, 13607, 13606, 13604, 13606, 13605, 13605, 13606, 13607, 13605, 13605, 13606, 13606, 13607, 13605, 13605, 13608, 13607, 13608, 13608, 13607, 13606, 13607, 13608, 13605, 13604, 13604, 13607, 13606, 13607, 13605, 13602, 13605, 13604, 13604, 13604, 13607, 13606, 13604, 13606, 13609, 13606, 13607, 13606, 13607, 13608, 13607, 13604, 13607, 13608, 13605, 13604, 13608, 13607, 13607, 13608, 13609, 13606, 13605, 13607, 13608, 13606, 13606, 13607, 13607, 13609, 13607, 13606, 13608, 13605, 13604, 13601, 13603, 13605, 13606, 13607, 13607, 13604, 13609, 13607, 13606, 13606, 13608, 13605, 13606, 13605, 13604, 13602, 13606, 13608, 13606, 13605, 13606, 13608, 13606, 13608, 13605, 13605, 13606, 13606, 13607, 13606, 13606, 13606, 13607, 13608, 13605, 13607, 13609, 13607, 13606, 13605, 13606, 13608, 13607, 13604, 13608, 13608, 13609, 13607, 13606, 13608, 13608, 13604, 13605, 13607, 13607, 13605, 13607, 13605, 13607, 13605, 13605, 13606, 13606, 13606, 13606, 13605, 13604, 13606, 13607, 13606, 13606, 13606, 13606, 13608, 13608, 13605, 13610, 13604, 13604, 13605, 13609, 13605, 13607, 13605, 13605, 13605, 13605, 13605, 13604, 13605, 13608, 13608, 13607, 13607, 13608, 13609, 13609, 13606, 13605, 13608, 13608, 13606, 13607, 13606, 13606, 13605, 13605, 13603, 13607, 13607, 13604, 13607, 13609, 13605, 13609, 13607, 13609, 13609, 13610, 13606, 13605, 13607, 13609, 13606, 13606, 13605, 13607, 13605, 13606, 13605, 13606, 13605, 13605, 13602, 13605, 13605, 13607, 13605, 13609, 13605, 13606, 13607, 13606, 13606, 13608, 13607, 13607, 13605, 13606, 13606, 13607, 13607, 13606, 13608, 13606, 13606, 13607, 13605, 13605, 13608, 13608, 13607, 13608, 13609, 13608, 13607, 13608, 13605, 13606, 13609, 13608, 13606, 13609, 13606, 13607, 13608, 13608, 13606, 13607, 13606, 13606, 13607, 13607, 13608, 13605, 13608, 13606, 13609, 13609, 13606, 13606, 13610, 13608, 13606, 13610, 13607, 13608, 13606, 13607, 13608, 13606, 13607, 13608, 13606, 13608, 13606, 13605, 13608, 13608, 13607, 13607, 13606, 13607, 13605, 13608, 13605, 13607, 13605, 13608, 13606, 13607, 13608, 13609, 13607, 13607, 13607, 13605, 13606, 13608, 13606, 13608, 13608, 13606, 13605, 13607, 13606, 13608, 13608, 13606, 13606, 13608, 13605, 13607, 13608, 13607, 13605, 13606, 13608, 13609, 13607, 13608, 13605, 13605, 13606, 13606, 13607, 13607, 13607, 13606, 13605, 13606, 13606, 13607, 13607, 13607, 13607, 13607, 13606, 13606, 13606, 13608, 13606, 13607, 13608, 13607, 13605, 13607, 13606, 13605, 13606, 13606, 13606, 13605, 13604, 13608, 13608, 13605, 13607, 13605, 13607, 13608, 13607, 13606, 13605, 13606, 13607, 13607, 13604, 13608, 13608, 13608, 13606, 13605, 13604, 13608, 13607, 13604, 13606, 13607, 13604, 13605, 13608, 13608, 13605, 13606, 13607, 13605, 13604, 13608, 13607, 13606, 13608, 13606, 13605, 13605, 13609, 13605, 13607, 13607, 13604, 13606, 13609, 13606, 13607, 13606, 13606, 13607, 13604, 13608, 13607, 13608, 13603, 13607, 13606, 13607, 13604, 13607, 13606, 13607, 13605, 13605, 13608, 13608, 13606, 13607, 13608, 13609, 13608, 13608, 13607, 13608, 13605, 13609, 13605, 13607, 13610, 13607, 13605, 13607, 13609, 13610, 13608, 13609, 13606, 13607]),
], dtype=rec_type
)
dt = rr_old_sample['dt'][0]*1e-9
rr_samples = len(rr_old_sample['data'][0])
num_rr_available = len(rr_old_sample[rr_old_sample['channel'] == 0])
rr_per_wf = np.int(math.ceil(98304/rr_samples))
num_events_available = np.int(num_rr_available/rr_per_wf)
last_rr_omitted = 5
wf_max_len = (rr_per_wf-last_rr_omitted)*rr_samples*dt
num_events_processed = num_events_available
wf_len = wf_max_len
wf_min_len = rr_samples*10e-9
records_required = np.int(math.ceil(wf_len/(rr_samples*dt)))
@numba.njit(cache=True, nogil=True)
def concatinate(rr_old_short, waveform_data, rr_samples, rr_per_wf, num_events_processed, num_events_available, records_required, active_channels, rr_dtype):
# Concatinate waveforms from raw_records: this is the core of the function
# Create an array of evenly spaced values in a given interval
events = np.arange(0,num_events_available,1) # we use the num available because it is the maximum
rr = [] #list of events, may need to convert to numpy array for faster process
#rr_wf = []
# Let's loop now on the events we want to process
for i in range(num_events_processed):
event = events[i]
if event == 0:
rr_start = 0
rr_stop = records_required - 1
else:
rr_start = np.int(event*rr_per_wf + events[i-1])
rr_stop = np.int((rr_start + records_required))
rr_s = np.zeros(len(active_channels), rr_dtype)
for ix,ch in enumerate(active_channels):
single_channel = rr_old_short[rr_old_short['channel'] == ch]
if event == 0:
baseline = np.median(waveform_data[rr_old_short['channel'] == ch][rr_start:records_required])
#baseline + flip waveform
long_wf = np.concatenate(int(baseline) - waveform_data[rr_old_short['channel'] == ch][rr_start:records_required])
else:
baseline = np.median(waveform_data[rr_old_short['channel'] == ch][rr_start:records_required])
#baseline + flip waveform
long_wf = np.concatenate(int(baseline) - waveform_data[rr_old_short['channel'] == ch][rr_start:rr_stop])
rr_s[ix]['baseline'] = np.median(baseline)
rr_s[ix]['time'] = single_channel[rr_start]['time']
rr_s[ix]['data'] = long_wf
rr_s[ix]['event'] = event
rr_s[ix]['channel'] = ch
rr_s[ix]['dt'] = 10
if ch <= 493:
rr_s[ix]['search'] = "dm"
else:
rr_s[ix]['search'] = "0vbb"
rr.append(rr_s)
return rr
rr_old_short = rr_old_sample[['time', 'length', 'dt', 'channel', 'pulse_length', 'record_i','baseline']]
waveform_data = rr_old_sample['data']
rr_dtype = np.dtype([('data',np.int,records_required*rr_samples),
('time',np.int),
('search',('U', 5)),
('channel',np.int),
('event',np.int),
('dt',np.int),
('baseline',np.float)
])
active_channel_small = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.int16)
wf_c = concatinate(rr_old_short,waveform_data,rr_samples,rr_per_wf,num_events_processed,num_events_available,records_required,active_channel_small, rr_dtype)
The error is:
TypingError: Failed in nopython mode pipeline (step: nopython frontend)
No implementation of function Function(<function concatenate at 0x7f60fc6ac700>) found for signature:
>>> concatenate(array(int64, 2d, C))
There are 2 candidate implementations:
- Of which 2 did not match due to:
Overload in function 'concatenate': File: numba/core/typing/npydecl.py: Line 792.
With argument(s): '(array(int64, 2d, C))':
Rejected as the implementation raised a specific error:
TypeError: np.concatenate(): expecting a non-empty tuple of arrays, got array(int64, 2d, C)
raised from /dali/lgrandi/strax/miniconda3/envs/strax/lib/python3.8/site-packages/numba/core/typing/npydecl.py:768
During: resolving callee type: Function(<function concatenate at 0x7f60fc6ac700>)
During: typing of call at <ipython-input-2-6a75c6aa7a9f> (67)
File "<ipython-input-2-6a75c6aa7a9f>", line 67:
def concatinate(rr_old_short, waveform_data, rr_samples, rr_per_wf, num_events_processed, num_events_available, records_required, active_channels, rr_dtype):
<source elided>
#baseline + flip waveform
long_wf = np.concatenate(int(baseline) - waveform_data[rr_old_short['channel'] == ch][rr_start:records_required])
^
I am sorry, would you be able to help me again?
Cheers,
Serena