3from __future__
import annotations
5Author: J.A. de Jong - ASCEE
7Description: Measurement class
9The ASCEE hdf5 measurement file format contains the following fields:
13'version': If not given, version 1 is assumed. For version 1, measurement data
14is assumed to be acoustic data.
15'samplerate': The audio data sample rate in Hz.
16'nchannels': The number of audio channels in the file
17'sensitivity': (Optionally) the stored sensitivity of the record channels.
18 This can be a single value, or a list of sensitivities for
19 each channel. Both representations are allowed.
21For measurement files of LASP < v1.0
22'qtys' : (Optionally): list of quantities that is recorded for each channel',
23if this array is not found. Quantities are defaulted to 'Number / Full scale'
25For measurement files of LASP >= 1.0
29'audio': 3-dimensional array of blocks of audio data. The first axis is the
30block index, the second axis the sample number and the third axis is the
31channel number. The data type is either int16, int32 or float64 / float32. If
32raw data is stored as integer values (int16, int32), the actual values should
33be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the
34corresponding 'number' lies between -1.0 and 1.0.
36'video': 4-dimensional array of video frames. The first index is the frame
37 number, the second the x-value of the pixel and the third is the
38 y-value of the pixel. Then, the last axis is the color. This axis has
39 length 3 and the colors are stored as (r,g,b). Where typically a
40 color depth of 256 is used (np.uint8 data format)
42The video dataset can possibly be not present in the data.
46__all__ = [
'Measurement',
'scaleBlockSens']
47from contextlib
import contextmanager
50from .lasp_config
import LASP_NUMPY_FLOAT_TYPE
51from scipy.io
import wavfile
52import os, time, wave, logging
53from .lasp_common
import SIQtys, Qty, getFreq
54from .lasp_cpp
import Window, DaqChannel, LASP_VERSION_MAJOR, AvPowerSpectra
55from typing
import List
56from functools
import lru_cache
60 """Returns the width of a single sample in **bytes**.
66 Size of a sample in bytes (int)
68 if dtype
in (np.int32, np.float32):
70 elif dtype == np.int16:
72 elif dtype == np.float64:
75 raise ValueError(
'Invalid data type: %s' % dtype)
79 """Scale a block of raw data to return raw acoustic pressure data.
82 block: block of raw data with integer data type
83 sens: array of sensitivity coeficients for
86 sens = np.asarray(sens)
87 assert sens.size == block.shape[1]
88 if np.issubdtype(block.dtype.type, np.integer):
90 fac = 2**(8 * sw - 1) - 1
93 return block.astype(LASP_NUMPY_FLOAT_TYPE) / fac / sens[np.newaxis, :]
97 """Iterate over stored blocks if the raw measurement data of a h5 file."""
100 """Initialize a BlockIter object.
103 f: Audio dataset in the h5 file, accessed as f['audio']
104 channels: list of channel indices to use
105 istart: index of first sample
106 istop: index of last sample (not including istop)
109 assert isinstance(channels, list)
114 nblocks = fa.shape[0]
115 blocksize = fa.shape[1]
121 self.
istop = kwargs.pop(
'istop', blocksize*nblocks)
125 if self.
istop % blocksize == 0:
131 self.
istop += blocksize*nblocks
133 raise ValueError(
'Stop index is smaller than start index')
135 if self.
istop != blocksize*nblocks:
144 """Return the next block."""
166 return fa[block, start_offset:stop_offset, :][:, self.
channels]
171 Iterate over blocks of data, scaled with sensitivity and integer scaling
174 def __init__(self, fa, channels, sensitivity, **kwargs):
175 super().
__init__(fa, channels, **kwargs)
177 assert self.
sens.ndim == 1
185 """Provides access to measurement data stored in the h5 measurement file
189 """Initialize a Measurement object based on the filename."""
201 with h5.File(fn,
'r')
as f:
210 dtype = f[
'audio'].dtype
232 logging.info(
"Measurement file obtained which stores channel names with *old* attribute 'channel_names'")
235 logging.info(
'No channel name data found in measurement')
239 if 'comment' in f.attrs:
246 sens = f.attrs[
'sensitivity']
249 sens, float)
else sens
262 qtys_enum_idx = f.attrs[
'qtys_enum_idx']
263 self.
_qtys = [SIQtys.fromInt(idx)
for idx
in qtys_enum_idx]
266 qtys_json = f.attrs[
'qtys']
268 self.
_qtys = [Qty.from_json(qty_json)
for qty_json
in qtys_json]
274 if self.
_qtys is None:
276 logging.debug(f
'Physical quantity data not available in measurement file. Assuming {SIQtys.default}')
280 Set an attribute in the measurement file, and keep a local copy in
281 memory for efficient accessing.
283 with self.
file(
'r+')
as f:
285 f.attrs[atrname] = value
286 setattr(self,
'_' + atrname, value)
290 """Returns filename base without extension."""
291 return os.path.splitext(self.
fn_base)[0]
300 raise RuntimeError(
'Invalid length of new channel names')
311 ch.sensitivity = sens
312 ch.qty = qty.cpp_enum
316 @channelConfig.setter
322 chname.append(ch.name)
323 sens.append(ch.sensitivity)
324 qtys.append(SIQtys.fromCppEnum(ch.qty))
336 if not len(newqtys) == len(self.
_qtys):
337 raise ValueError(
'Invalid number of quantities')
338 qtys_int = [qty.toInt()
for qty
in newqtys]
342 with self.
file(
'r+')
as f:
344 f.attrs[
'qtys_enum_idx'] = qtys_int
350 """Contextmanager which opens the storage file and yields the file.
353 mode: Opening mode for the file. Should either be 'r', or 'r+'
355 if mode
not in (
'r',
'r+'):
356 raise ValueError(
'Invalid file opening mode.')
357 with h5.File(self.
fn, mode)
as f:
362 """Return the measurement comment.
365 The measurement comment (text string)
371 """Set the measurement comment.
374 cmt: Comment text string to set
376 with self.
file(
'r+')
as f:
378 f.attrs[
'comment'] = cmt
384 """Returns the total recording time of the measurement, in float
390 """Returns the measurement time in seconds since the epoch."""
397 Return a properly formatted string of the measurement time, in order of
399 year-month-day hour etc.
402 time_struct = time.localtime(self.
time)
403 time_string = time.strftime(
'%Y-%m-%d %H:%M:%S', time_struct)
406 def rms(self, channels=None, substract_average=False):
407 """Returns the root mean square values for each channel
410 channels: list of channels
411 substract_average: If set to true, computes the rms of only the
412 oscillating component of the signal, which is in fact the signal
416 1D array with rms values for each channel
421 with self.
file()
as f:
422 for block
in self.
iterData(channels):
423 Nblock = block.shape[0]
424 sum_ += np.sum(block, axis=0)
426 meansquare += np.sum(block**2, axis=0) / self.
N
431 if substract_average:
433 rms = np.sqrt(meansquare)
437 return self.
rms(substract_average=
True)
440 """Returns the raw data as stored in the measurement file,
441 without any transformations applied
444 channels: list, or tuple of channel numbers to export. If not defined, all
445 channels in the measurement are returned
448 Numpy array with data. The first axis is always the time instance,
449 the second axis the channel number.
457 with self.
file()
as f:
459 rawdata.append(block)
461 return np.concatenate(rawdata, axis=0)
467 with self.
file()
as f:
468 for block
in IterData(f, channels, sensitivity, **kwargs):
471 def data(self, channels=None, **kwargs):
473 Returns the measurement data, scaled and sensitivity applied.
476 for d
in self.
iterData(channels, **kwargs):
479 return np.concatenate(data, axis=0)
481 def CPS(self, channels=None, **kwargs):
483 Compute single-sided Cross-Power-Spectrum of the measurement channels
486 channels: Channels to compute for (numbers)
491 overlap: Overlap percentage (value between 0.0 and up to and
496 Cross-power-spectra. C[freq, ch_i, ch_j] = C_ij
499 nfft = kwargs.pop(
'nfft', 2048)
500 window = kwargs.pop(
'windowType', Window.WindowType.Hann)
501 overlap = kwargs.pop(
'overlap', 50.0)
506 nchannels = len(channels)
507 aps = AvPowerSpectra(nfft, window, overlap)
510 for data
in self.
iterData(channels, **kwargs):
511 CS = aps.compute(data)
513 return freq, aps.get_est()
517 Return the (coherent) periodic average the measurement. This method is
518 useful for a situation of periodic excitation.
521 N: The number of samples in one period. This value should
522 correspond with the period of the excitation!
523 noiseCorrection: whether to apply coherent averaging, according to
524 the Sliding Window correlation method (SWiC): Telle et al.: A Novel
525 Approach for Impulse Response Measurements with Time-Varying Noise.
526 If set to False, just the arithmetic average is used.
534 signal = self.
data(channels)
537 blocks = [signal[i*N:(i+1)*N]
for i
in range(Nblocks)]
542 en = [
None] + [blocks[i] - blocks[i-1]
for i
in range(1,Nblocks)]
544 noise_est = [
None] + [-np.average(en[i]*en[i+1])
for i
in range(1,len(en)-1)]
547 sum_inverse_noise = sum([1/n
for n
in noise_est[1:]])
548 c_n = [1/(ni*sum_inverse_noise)
for ni
in noise_est[1:]]
550 c_n = [1/(Nblocks-2)]*(Nblocks-2)
552 assert np.isclose(sum(c_n), 1.0)
553 assert Nblocks-2 == len(c_n)
556 avg = np.zeros((blocks[0].shape), dtype=float)
557 for n
in range(0, Nblocks-2):
558 avg += c_n[n]*blocks[n+1]
564 Compute Cross-Spectral Density based on periodic excitation. Uses noise
565 reduction by time-averaging the data.
571 nchannels = len(channels)
572 window = Window.rectangular
584 """Sensitivity of the data in U^-1, from floating point data scaled
585 between -1.0 and 1.0 to Units [U].
587 If the sensitivity is not stored in the measurement file, this
588 function returns 1.0 for each channel
594 """Set the sensitivity of the measurement in the file.
597 sens: sensitivity data, should be a float, or an array of floats
598 equal to the number of channels.
600 if isinstance(sens, float):
603 elif isinstance(sens, list):
604 sens = np.asarray(sens)
606 valid = sens.ndim == 1
608 valid &= sens.dtype == float
610 raise ValueError(
'Invalid sensitivity value(s) given')
611 with self.
file(
'r+')
as f:
612 f.attrs[
'sensitivity'] = sens
616 """Coarse check for overflow in measurement.
619 True if overflow is possible, else False
622 for block
in self.
iterData(channels):
624 if dtype.kind ==
'i':
626 maxvalue = np.iinfo(dtype).max
627 if np.max(np.abs(block)) >= 0.9*maxvalue:
636 normalize=False, **kwargs):
637 """Export measurement file as wave. In case the measurement data is
638 stored as floats, the values are scaled to the proper integer (PCM)
642 fn: If given, this will be the filename to write to. If the
643 filename does not end with '.wav', this extension is added.
645 force: If True, overwrites any existing files with the given name
646 , otherwise a RuntimeError is raised.
648 dtype: if not None, convert data to this data type.
649 Options are 'int16', 'int32', 'float32'.
651 normalize: If set: normalize the level to something sensible.
655 fn = os.path.splitext(fn)[0]
657 if os.path.splitext(fn)[1] !=
'.wav':
660 if os.path.exists(fn)
and not force:
661 raise RuntimeError(f
'File already exists: {fn}')
664 raise RuntimeError(f
'Sample rates should be approximately integer for exporting to Wave to work')
673 maxabs = np.max(np.abs(data))
678 logging.debug(f
"dtype not passed as arg; using dtype = {dtype}")
687 elif dtype==
'float32':
689 elif dtype==
'float64':
692 logging.debug(f
"cannot handle this dtype {dtype}")
700 if dtype==
'int16' or dtype==
'int32':
702 scalefac = 2**(8*newsampwidth-1)-1
703 data = (data*scalefac).astype(newtype)
705 wavfile.write(fn, int(self.
samplerate), data.astype(newtype))
716 """Converts a txt file to a LASP Measurement file, opens the associated
717 Measurement object and returns it. The measurement file will have the
718 same file name as the txt file, except with h5 extension.
721 fn: Filename of text file
722 skiprows: Number of header rows in text file to skip
723 samplerate: Sampling frequency in [Hz]
724 sensitivity: 1D array of channel sensitivities
725 mfn: Filepath where measurement file is stored. If not given,
726 a h5 file will be created along fn, which shares its basename
727 timestamp: If given, a custom timestamp for the measurement
728 (integer containing seconds since epoch). If not given, the
729 timestamp is obtained from the last modification time.
730 delimiter: Column delimiter
731 firstcoltime: If true, the first column is the treated as the
734 if not os.path.exists(fn):
735 raise ValueError(f
'File {fn} does not exist.')
736 if timestamp
is None:
737 timestamp = os.path.getmtime(fn)
739 mfn = os.path.splitext(fn)[0] +
'.h5'
741 mfn = os.path.splitext(mfn)[0] +
'.h5'
743 dat = np.loadtxt(fn, skiprows=skiprows, delimiter=delimiter)
746 if not np.isclose(time[1] - time[0], 1 / samplerate):
747 raise ValueError(
'Samplerate given does not agree with '
748 'samplerate in file')
752 nchannels = dat.shape[1]
753 if nchannels != sensitivity.shape[0]:
755 f
'Invalid sensitivity length given. Should be: {nchannels}')
757 with h5.File(mfn,
'w')
as hf:
758 hf.attrs[
'samplerate'] = samplerate
759 hf.attrs[
'sensitivity'] = sensitivity
760 hf.attrs[
'time'] = timestamp
761 hf.attrs[
'blocksize'] = 1
762 hf.attrs[
'nchannels'] = nchannels
763 ad = hf.create_dataset(
'audio', (1, dat.shape[0], dat.shape[1]),
765 maxshape=(1, dat.shape[0], dat.shape[1]),
776 qtys: List[SIQtys] =
None,
777 channelNames: List[str] =
None,
778 force=
False) -> Measurement:
780 Creates a LASP measurement file from input numpy array data.
781 Opens the associated Measurement object and returns it.
784 data: Numpy array, first column is sample, second is channel. Can
785 also be specified with a single column for single-channel data.
787 samplerate: Sampling frequency in [Hz]
789 sensitivity: 1D array of channel sensitivities in [U^-1], where U is
792 mfn: Filepath of the file where the data is stored.
794 timestamp: If given, a custom timestamp for the measurement
795 (integer containing seconds since epoch).
797 qtys: If a list of physical quantity data is given here
799 channelNames: Name of the channels
801 force: If True, overwrites existing files with specified `mfn`
804 if os.path.splitext(mfn)[1] !=
'.h5':
806 if os.path.exists(mfn)
and not force:
807 raise ValueError(f
'File {mfn} already exist.')
808 if timestamp
is None:
809 timestamp = int(time.time())
812 data = data[:, np.newaxis]
818 raise ValueError(
'Sensitivity should be given as array-like data type')
819 sensitivity = np.asarray(sensitivity)
821 nchannels = data.shape[1]
822 if nchannels != sensitivity.shape[0]:
824 f
'Invalid sensitivity length given. Should be: {nchannels}')
826 if channelNames
is not None:
827 if len(channelNames) != nchannels:
828 raise RuntimeError(
"Illegal length of channelNames list given")
831 qtys = [SIQtys.AP]*nchannels
833 if len(qtys) != nchannels:
834 raise RuntimeError(
"Illegal length of qtys list given")
836 qtyvals = [qty.value
for qty
in qtys]
838 with h5.File(mfn,
'w')
as hf:
839 hf.attrs[
'samplerate'] = samplerate
840 hf.attrs[
'sensitivity'] = sensitivity
841 hf.attrs[
'time'] = timestamp
842 hf.attrs[
'blocksize'] = 1
843 hf.attrs[
'nchannels'] = nchannels
846 hf.attrs[
'qtys_enum_idx'] = [qtyval.toInt()
for qtyval
in qtyvals]
849 if channelNames
is not None:
850 hf.attrs[
'channelNames'] = channelNames
852 ad = hf.create_dataset(
'audio', (1, data.shape[0], data.shape[1]),
854 maxshape=(1, data.shape[0], data.shape[1]),
861 """Convert a measurement file to a wave file, and return the
862 measurement handle."""
863 if timestamp
is None:
864 timestamp = int(time.time())
866 base_fn = os.path.splitext(fn)[0]
868 newfn = base_fn +
'.h5'
869 if os.path.exists(newfn)
and not force:
870 raise RuntimeError(f
'Measurement file name {newfn} already exists in path, set "force" to true to overwrite')
872 samplerate, data = wavfile.read(fn)
874 nframes, nchannels = data.shape
878 data = data[:, np.newaxis]
879 sensitivity = np.ones(nchannels)
881 with h5.File(newfn,
'w')
as hf:
882 hf.attrs[
'samplerate'] = samplerate
883 hf.attrs[
'nchannels'] = nchannels
884 hf.attrs[
'time'] = timestamp
885 hf.attrs[
'blocksize'] = 1
886 hf.attrs[
'sensitivity'] = sensitivity
887 ad = hf.create_dataset(
'audio', (1, nframes, nchannels),
889 maxshape=(1, nframes, nchannels),
Computes single-sided cross-power spectra for a group of channels. Only a single block of length fft,...
Iterate over blocks of data, scaled with sensitivity and integer scaling between 0 and 1.
__next__(self)
Return the next block.
__init__(self, fa, channels, sensitivity, **kwargs)
Initialize a BlockIter object.
Iterate over stored blocks if the raw measurement data of a h5 file.
__init__(self, f, channels, **kwargs)
Initialize a BlockIter object.
__next__(self)
Return the next block.
Provides access to measurement data stored in the h5 measurement file format.
sensitivity(self, sens)
Set the sensitivity of the measurement in the file.
periodicAverage(self, N, channels=None, noiseCorrection=True, **kwargs)
Return the (coherent) periodic average the measurement.
exportAsWave(self, fn=None, force=False, dtype=None, normalize=False, **kwargs)
Export measurement file as wave.
file(self, mode='r')
Contextmanager which opens the storage file and yields the file.
name(self)
Returns filename base without extension.
channelNames(self, newchnames)
fromtxt(fn, skiprows, samplerate, sensitivity, mfn=None, timestamp=None, delimiter='\t', firstcoltime=True)
Converts a txt file to a LASP Measurement file, opens the associated Measurement object and returns i...
checkOverflow(self, channels)
Coarse check for overflow in measurement.
recTime(self)
Returns the total recording time of the measurement, in float seconds.
rms(self, channels=None, substract_average=False)
Returns the root mean square values for each channel.
periodicCPS(self, N, channels=None, **kwargs)
Compute Cross-Spectral Density based on periodic excitation.
rawData(self, channels=None, **kwargs)
Returns the raw data as stored in the measurement file, without any transformations applied.
sensitivity(self)
Sensitivity of the data in U^-1, from floating point data scaled between -1.0 and 1....
__init__(self, fn)
Initialize a Measurement object based on the filename.
timestr(self)
Return a properly formatted string of the measurement time, in order of.
Measurement fromnpy(data, samplerate, sensitivity, mfn, timestamp=None, List[SIQtys] qtys=None, List[str] channelNames=None, force=False)
Creates a LASP measurement file from input numpy array data.
CPS(self, channels=None, **kwargs)
Compute single-sided Cross-Power-Spectrum of the measurement channels.
comment(self)
Return the measurement comment.
time(self)
Returns the measurement time in seconds since the epoch.
data(self, channels=None, **kwargs)
Returns the measurement data, scaled and sensitivity applied.
variance(self, channels=None)
setAttribute(self, atrname, value)
Set an attribute in the measurement file, and keep a local copy in memory for efficient accessing.
fromWaveFile(fn, newfn=None, force=False, timestamp=None)
Convert a measurement file to a wave file, and return the measurement handle.
iterData(self, channels, **kwargs)
getSampWidth(dtype)
Returns the width of a single sample in bytes.
scaleBlockSens(block, sens)
Scale a block of raw data to return raw acoustic pressure data.