diff --git a/.gitignore b/.gitignore index dedd7f754..fcf8682c7 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,11 @@ external/ ci/coverage.info ci/html/ +# Files produced by testing suite +invalid_points.xml +test_att.h5 +test_dim1.h5 +test_dim2.h5 +test1_001.h5 +testing_0.5 +valid_points.xml diff --git a/ADApp/ADSrc/Codec.h b/ADApp/ADSrc/Codec.h index d22ab3ff1..db54cb493 100644 --- a/ADApp/ADSrc/Codec.h +++ b/ADApp/ADSrc/Codec.h @@ -1,9 +1,12 @@ #ifndef Codec_H #define Codec_H +#include + static std::string codecName[] = { "", "jpeg", + "zlib", "blosc", "lz4", "lz4hdf5", @@ -13,6 +16,7 @@ static std::string codecName[] = { typedef enum { NDCODEC_NONE, NDCODEC_JPEG, + NDCODEC_ZLIB, NDCODEC_BLOSC, NDCODEC_LZ4, NDCODEC_LZ4HDF5, diff --git a/ADApp/Db/NDCodec.template b/ADApp/Db/NDCodec.template index 3f3eb47de..03c4dd503 100644 --- a/ADApp/Db/NDCodec.template +++ b/ADApp/Db/NDCodec.template @@ -38,14 +38,16 @@ record(mbbo, "$(P)$(R)Compressor") field(ZRVL, "0") field(ONST, "JPEG") field(ONVL, "1") - field(TWST, "Blosc") + field(TWST, "Zlib") field(TWVL, "2") - field(THST, "LZ4") + field(THST, "Blosc") field(THVL, "3") - field(FRST, "LZ4HDF5") + field(FRST, "LZ4") field(FRVL, "4") - field(FVST, "BSLZ4") + field(FVST, "LZ4HDF5") field(FVVL, "5") + field(SXST, "BSLZ4") + field(SXVL, "6") info(autosaveFields, "VAL") } @@ -57,14 +59,16 @@ record(mbbi, "$(P)$(R)Compressor_RBV") field(ZRVL, "0") field(ONST, "JPEG") field(ONVL, "1") - field(TWST, "Blosc") + field(TWST, "Zlib") field(TWVL, "2") - field(THST, "LZ4") + field(THST, "Blosc") field(THVL, "3") - field(FRST, "LZ4HDF5") + field(FRST, "LZ4") field(FRVL, "4") - field(FVST, "BSLZ4") + field(FVST, "LZ4HDF5") field(FVVL, "5") + field(SXST, "BSLZ4") + field(SXVL, "6") field(SCAN, "I/O Intr") } @@ -95,6 +99,24 @@ record(longin, "$(P)$(R)JPEGQuality_RBV") field(SCAN, "I/O Intr") } +record(longout, "$(P)$(R)ZlibCLevel") +{ + field(PINI, "YES") + field(DTYP, "asynInt32") + field(OUT, "@asyn($(PORT),$(ADDR=0),$(TIMEOUT=1))ZLIB_CLEVEL") + field(VAL, "6") + field(DRVH, "9") + field(DRVL, "0") + info(autosaveFields, "VAL") +} + +record(longin, "$(P)$(R)ZlibCLevel_RBV") +{ + field(DTYP, "asynInt32") + field(INP, "@asyn($(PORT),$(ADDR=0),$(TIMEOUT=1))ZLIB_CLEVEL") + field(SCAN, "I/O Intr") +} + record(mbbo, "$(P)$(R)BloscCompressor") { field(PINI, "YES") diff --git a/ADApp/Db/NDCodec_settings.req b/ADApp/Db/NDCodec_settings.req index 8c2a34986..9981e2c50 100644 --- a/ADApp/Db/NDCodec_settings.req +++ b/ADApp/Db/NDCodec_settings.req @@ -1,6 +1,7 @@ $(P)$(R)Mode $(P)$(R)Compressor $(P)$(R)JPEGQuality +$(P)$(R)ZlibCLevel $(P)$(R)BloscCompressor $(P)$(R)BloscCLevel $(P)$(R)BloscShuffle diff --git a/ADApp/op/adl/NDCodec.adl b/ADApp/op/adl/NDCodec.adl index 1964c812f..01362dc67 100644 --- a/ADApp/op/adl/NDCodec.adl +++ b/ADApp/op/adl/NDCodec.adl @@ -1,12 +1,12 @@ file { - name="/home/epics/devel/areaDetector-3-14/ADCore/ADApp/op/adl/NDCodec.adl" - version=030117 + name="/home/jwlodek/Workspace/ADCore/ADApp/op/adl/NDCodec.adl" + version=030122 } display { object { - x=330 - y=189 + x=3193 + y=313 width=775 height=600 } @@ -116,7 +116,7 @@ rectangle { x=390 y=40 width=380 - height=285 + height=310 } "basic attribute" { clr=14 @@ -263,7 +263,7 @@ text { "text update" { object { x=665 - y=151 + y=175 width=100 height=18 } @@ -279,7 +279,7 @@ text { "text update" { object { x=665 - y=176 + y=200 width=100 height=18 } @@ -294,7 +294,7 @@ text { "text update" { object { x=665 - y=201 + y=225 width=100 height=18 } @@ -310,7 +310,7 @@ text { menu { object { x=580 - y=151 + y=175 width=80 height=18 } @@ -323,7 +323,7 @@ menu { "text entry" { object { x=580 - y=175 + y=199 width=80 height=20 } @@ -338,7 +338,7 @@ menu { menu { object { x=580 - y=201 + y=225 width=80 height=18 } @@ -351,7 +351,7 @@ menu { text { object { x=415 - y=150 + y=174 width=160 height=20 } @@ -364,7 +364,7 @@ text { text { object { x=405 - y=175 + y=199 width=170 height=20 } @@ -377,7 +377,7 @@ text { text { object { x=445 - y=200 + y=224 width=130 height=20 } @@ -418,7 +418,7 @@ text { text { object { x=455 - y=275 + y=299 width=120 height=20 } @@ -431,7 +431,7 @@ text { "text update" { object { x=580 - y=276 + y=300 width=80 height=18 } @@ -449,7 +449,7 @@ text { text { object { x=395 - y=300 + y=325 width=110 height=20 } @@ -462,7 +462,7 @@ text { "text update" { object { x=510 - y=303 + y=328 width=255 height=14 } @@ -478,7 +478,7 @@ text { "text update" { object { x=665 - y=226 + y=250 width=100 height=18 } @@ -493,7 +493,7 @@ text { "text entry" { object { x=580 - y=225 + y=249 width=80 height=20 } @@ -508,7 +508,7 @@ text { text { object { x=405 - y=225 + y=249 width=170 height=20 } @@ -521,7 +521,7 @@ text { "text update" { object { x=664 - y=251 + y=275 width=100 height=18 } @@ -536,7 +536,7 @@ text { "text entry" { object { x=579 - y=250 + y=274 width=80 height=20 } @@ -551,7 +551,7 @@ text { text { object { x=394 - y=250 + y=274 width=180 height=20 } @@ -561,3 +561,46 @@ text { textix="LZ4HDF5 Block Size" align="horiz. right" } +text { + object { + x=455 + y=150 + width=120 + height=20 + } + "basic attribute" { + clr=14 + } + textix="Zlib Comp. Level" + align="horiz. right" +} +"text entry" { + object { + x=580 + y=150 + width=80 + height=20 + } + control { + chan="$(P)$(R)ZlibCLevel" + clr=14 + bclr=51 + } + limits { + } +} +"text update" { + object { + x=665 + y=151 + width=100 + height=18 + } + monitor { + chan="$(P)$(R)ZlibCLevel_RBV" + clr=54 + bclr=4 + } + limits { + } +} diff --git a/ADApp/pluginSrc/Makefile b/ADApp/pluginSrc/Makefile index 9d3d2048e..d6bb7dbd6 100644 --- a/ADApp/pluginSrc/Makefile +++ b/ADApp/pluginSrc/Makefile @@ -211,6 +211,10 @@ ifeq ($(WITH_BITSHUFFLE), YES) USR_CXXFLAGS += -DHAVE_BITSHUFFLE endif +ifeq ($(WITH_ZLIB), YES) + USR_CXXFLAGS += -DHAVE_ZLIB +endif + ifdef BLOSC_INCLUDE USR_INCLUDES += $(addprefix -I, $(BLOSC_INCLUDE)) endif @@ -219,6 +223,10 @@ ifdef BITSHUFFLE_INCLUDE USR_INCLUDES += $(addprefix -I, $(BITSHUFFLE_INCLUDE)) endif +ifdef ZLIB_INCLUDE + USR_INCLUDES += $(addprefix -I, $(ZLIB_INCLUDE)) +endif + ifdef HDF5_INCLUDE USR_INCLUDES += $(addprefix -I, $(HDF5_INCLUDE)) endif diff --git a/ADApp/pluginSrc/NDFileHDF5.cpp b/ADApp/pluginSrc/NDFileHDF5.cpp index ff6163715..6a8c04f34 100644 --- a/ADApp/pluginSrc/NDFileHDF5.cpp +++ b/ADApp/pluginSrc/NDFileHDF5.cpp @@ -3323,6 +3323,9 @@ asynStatus NDFileHDF5::configureCompression(NDArray *pArray) setIntegerParam(NDFileHDF5_bloscCompressLevel, pArray->codec.level); setIntegerParam(NDFileHDF5_bloscShuffleType, pArray->codec.shuffle); setIntegerParam(NDFileHDF5_bloscCompressor, pArray->codec.compressor); + } else if (pArray->codec.name == codecName[NDCODEC_ZLIB]) { + setIntegerParam(NDFileHDF5_compressionType, HDF5CompressZlib); + setIntegerParam(NDFileHDF5_zCompressLevel, pArray->codec.level); } else if (pArray->codec.name == codecName[NDCODEC_BSLZ4]) { setIntegerParam(NDFileHDF5_compressionType, HDF5CompressBshuf); } else if (pArray->codec.name == codecName[NDCODEC_LZ4]) { @@ -3381,7 +3384,8 @@ asynStatus NDFileHDF5::configureCompression(NDArray *pArray) "%s::%s Setting zlib compression filter level=%d\n", driverName, functionName, zLevel); H5Pset_deflate(this->cparms, zLevel); - this->codec.name = "zlib"; + this->codec.name = codecName[NDCODEC_ZLIB]; + this->codec.level = zLevel; break; case HDF5CompressBlosc: { asynPrint(this->pasynUserSelf, ASYN_TRACE_FLOW, diff --git a/ADApp/pluginSrc/NDPluginCodec.cpp b/ADApp/pluginSrc/NDPluginCodec.cpp index 5ce79c872..75b3abcbd 100644 --- a/ADApp/pluginSrc/NDPluginCodec.cpp +++ b/ADApp/pluginSrc/NDPluginCodec.cpp @@ -343,6 +343,103 @@ NDArray *decompressJPEG(NDArray*, NDCodecStatus_t *status, char *errorMessage) #endif // ifdef HAVE_JPEG +#ifdef HAVE_ZLIB +#include + +NDArray *compressZlib(NDArray *input, int clevel, NDCodecStatus_t *status, char *errorMessage) +{ + if (!input->codec.empty()) { + sprintf(errorMessage, "Array is already compressed"); + *status = NDCODEC_WARNING; + return NULL; + } + + if (clevel < 0) clevel = 0; + if (clevel > 9) clevel = 9; + + NDArrayInfo_t info; + input->getInfo(&info); + + uLongf destLen = compressBound((uLong)info.totalBytes); + NDArray *output = allocArray(input, -1, destLen); + + if (!output) { + sprintf(errorMessage, "Failed to allocate zlib output array"); + *status = NDCODEC_ERROR; + return NULL; + } + + int ret = compress2((Bytef*)output->pData, &destLen, + (const Bytef*)input->pData, (uLong)info.totalBytes, clevel); + + if (ret != Z_OK) { + output->release(); + sprintf(errorMessage, "zlib compress2 failed with error %d", ret); + *status = NDCODEC_ERROR; + return NULL; + } + + output->codec.name = codecName[NDCODEC_ZLIB]; + output->codec.level = clevel; + output->compressedSize = (size_t)destLen; + + return output; +} + +NDArray *decompressZlib(NDArray *input, NDCodecStatus_t *status, char *errorMessage) +{ + if (input->codec.name != codecName[NDCODEC_ZLIB]) { + sprintf(errorMessage, "Invalid codec '%s', expected '%s'", + input->codec.name.c_str(), codecName[NDCODEC_ZLIB].c_str()); + *status = NDCODEC_ERROR; + return NULL; + } + + NDArrayInfo_t info; + input->getInfo(&info); + + NDArray *output = allocArray(input); + + if (!output) { + sprintf(errorMessage, "Failed to allocate zlib output array"); + *status = NDCODEC_ERROR; + return NULL; + } + + uLongf destLen = (uLongf)output->dataSize; + int ret = uncompress((Bytef*)output->pData, &destLen, + (const Bytef*)input->pData, (uLong)input->compressedSize); + + if (ret != Z_OK) { + output->release(); + sprintf(errorMessage, "zlib uncompress failed with error %d", ret); + *status = NDCODEC_ERROR; + return NULL; + } + + output->codec.clear(); + + return output; +} + +#else + +NDArray *compressZlib(NDArray*, int, NDCodecStatus_t *status, char *errorMessage) +{ + sprintf(errorMessage, "No zlib support"); + *status = NDCODEC_ERROR; + return NULL; +} + +NDArray *decompressZlib(NDArray*, NDCodecStatus_t *status, char *errorMessage) +{ + sprintf(errorMessage, "No zlib support"); + *status = NDCODEC_ERROR; + return NULL; +} + +#endif // ifdef HAVE_ZLIB + #ifdef HAVE_BLOSC #include @@ -780,6 +877,16 @@ void NDPluginCodec::processCallbacks(NDArray *pArray) break; } + case NDCODEC_ZLIB: { + int clevel; + getIntegerParam(NDCodecZlibCLevel, &clevel); + + unlock(); + result = compressZlib(pArray, clevel, &codecStatus, errorMessage); + lock(); + break; + } + case NDCODEC_BLOSC: { int numThreads, clevel, shuffle, compressor; @@ -836,6 +943,11 @@ void NDPluginCodec::processCallbacks(NDArray *pArray) result = decompressJPEG(pArray, &codecStatus, errorMessage); lock(); setIntegerParam(NDCodecCompressor, NDCODEC_JPEG); + } else if (pArray->codec.name == codecName[NDCODEC_ZLIB]) { + unlock(); + result = decompressZlib(pArray, &codecStatus, errorMessage); + lock(); + setIntegerParam(NDCodecCompressor, NDCODEC_ZLIB); } else if (pArray->codec.name == codecName[NDCODEC_BLOSC]) { int numThreads; getIntegerParam(NDCodecBloscNumThreads, &numThreads); @@ -901,6 +1013,11 @@ asynStatus NDPluginCodec::writeInt32(asynUser *pasynUser, epicsInt32 value) if (function == NDCodecJPEGQuality) { value = jpeg_clamp_quality(value); + } else if (function == NDCodecZlibCLevel) { + if (value < 0) + value = 0; + else if (value > 9) + value = 9; } else if (function == NDCodecBloscCompressor) { if (value < NDCODEC_BLOSC_BLOSCLZ) value = NDCODEC_BLOSC_BLOSCLZ; @@ -983,6 +1100,7 @@ NDPluginCodec::NDPluginCodec(const char *portName, int queueSize, int blockingCa createParam(NDCodecCodecStatusString, asynParamInt32, &NDCodecCodecStatus); createParam(NDCodecCodecErrorString, asynParamOctet, &NDCodecCodecError); createParam(NDCodecJPEGQualityString, asynParamInt32, &NDCodecJPEGQuality); + createParam(NDCodecZlibCLevelString, asynParamInt32, &NDCodecZlibCLevel); createParam(NDCodecBloscCompressorString, asynParamInt32, &NDCodecBloscCompressor); createParam(NDCodecBloscCLevelString, asynParamInt32, &NDCodecBloscCLevel); createParam(NDCodecBloscShuffleString, asynParamInt32, &NDCodecBloscShuffle); @@ -997,6 +1115,7 @@ NDPluginCodec::NDPluginCodec(const char *portName, int queueSize, int blockingCa setIntegerParam(NDCodecCodecStatus, NDCODEC_SUCCESS); setStringParam(NDCodecCodecError, ""); setIntegerParam(NDCodecJPEGQuality, 85); + setIntegerParam(NDCodecZlibCLevel, 6); setIntegerParam(NDCodecBloscCompressor, NDCODEC_BLOSC_BLOSCLZ); setIntegerParam(NDCodecBloscCLevel, 5); setIntegerParam(NDCodecBloscNumThreads, 1); diff --git a/ADApp/pluginSrc/NDPluginCodec.h b/ADApp/pluginSrc/NDPluginCodec.h index b6084ad3e..0682a411b 100644 --- a/ADApp/pluginSrc/NDPluginCodec.h +++ b/ADApp/pluginSrc/NDPluginCodec.h @@ -9,6 +9,7 @@ #define NDCodecCodecStatusString "CODEC_STATUS" /* (int r/o) Compression status: success or failure */ #define NDCodecCodecErrorString "CODEC_ERROR" /* (string r/o) Error message if compression fails */ #define NDCodecJPEGQualityString "JPEG_QUALITY" /* (int r/w) JPEG Compression quality */ +#define NDCodecZlibCLevelString "ZLIB_CLEVEL" /* (int r/w) Zlib compression level */ #define NDCodecBloscCompressorString "BLOSC_COMPRESSOR" /* (NDCodecBloscComp_t r/w) Which Blosc compressor to use */ #define NDCodecBloscCLevelString "BLOSC_CLEVEL" /* (int r/w) Blosc compression level */ #define NDCodecBloscShuffleString "BLOSC_SHUFFLE" /* (bool r/w) Should Blosc apply shuffling? */ @@ -51,6 +52,9 @@ typedef enum { NDArray *compressJPEG(NDArray *input, int quality, NDCodecStatus_t *status, char *errorMessage); NDArray *decompressJPEG(NDArray *input, NDCodecStatus_t *status, char *errorMessage); +NDArray *compressZlib(NDArray *input, int clevel, NDCodecStatus_t *status, char *errorMessage); +NDArray *decompressZlib(NDArray *input, NDCodecStatus_t *status, char *errorMessage); + NDArray *compressBlosc(NDArray *input, int clevel, int shuffle, NDCodecBloscComp_t compressor, int numThreads, NDCodecStatus_t *status, char *errorMessage); NDArray *decompressBlosc(NDArray *input, int numThreads, NDCodecStatus_t *status, char *errorMessage); @@ -81,6 +85,7 @@ class NDPLUGIN_API NDPluginCodec : public NDPluginDriver { int NDCodecCodecStatus; int NDCodecCodecError; int NDCodecJPEGQuality; + int NDCodecZlibCLevel; int NDCodecBloscCompressor; int NDCodecBloscCLevel; int NDCodecBloscShuffle; diff --git a/ADApp/pluginTests/CodecPluginWrapper.cpp b/ADApp/pluginTests/CodecPluginWrapper.cpp new file mode 100644 index 000000000..76b71141a --- /dev/null +++ b/ADApp/pluginTests/CodecPluginWrapper.cpp @@ -0,0 +1,36 @@ +/* + * CodecPluginWrapper.cpp + * + * Created on: 20 Apr 2026 + * Author: Jakub Wlodek + */ + +#include "CodecPluginWrapper.h" + +CodecPluginWrapper::CodecPluginWrapper(const std::string& port, const std::string& detectorPort) + : NDPluginCodec(port.c_str(), 50, 0, detectorPort.c_str(), 0, 0, 0, 0, 0, 1), + AsynPortClientContainer(port) +{ +} + +CodecPluginWrapper::CodecPluginWrapper(const std::string& port, + int queueSize, + int blocking, + const std::string& detectorPort, + int address, + size_t maxMemory, + int priority, + int stackSize, + int maxThreads) + : NDPluginCodec(port.c_str(), queueSize, blocking, + detectorPort.c_str(), address, + 0, maxMemory, priority, stackSize, maxThreads), + AsynPortClientContainer(port) +{ +} + +CodecPluginWrapper::~CodecPluginWrapper () +{ + cleanup(); +} + diff --git a/ADApp/pluginTests/CodecPluginWrapper.h b/ADApp/pluginTests/CodecPluginWrapper.h new file mode 100644 index 000000000..1efb77f7b --- /dev/null +++ b/ADApp/pluginTests/CodecPluginWrapper.h @@ -0,0 +1,30 @@ +/* + * CodecPluginWrapper.cpp + * + * Created on: 20 Apr 2026 + * Author: Jakub Wlodek + */ + +#ifndef ADAPP_PLUGINTESTS_CODECPLUGINWRAPPER_H_ +#define ADAPP_PLUGINTESTS_CODECPLUGINWRAPPER_H_ + +#include +#include "AsynPortClientContainer.h" + +class CodecPluginWrapper : public NDPluginCodec, public AsynPortClientContainer +{ +public: + CodecPluginWrapper(const std::string& port, const std::string& detectorPort); + CodecPluginWrapper(const std::string& port, + int queueSize, + int blocking, + const std::string& detectorPort, + int address, + size_t maxMemory, + int priority, + int stackSize, + int maxThreads); + virtual ~CodecPluginWrapper (); +}; + +#endif /* ADAPP_PLUGINTESTS_CODECPLUGINWRAPPER_H_ */ diff --git a/ADApp/pluginTests/FFTPluginWrapper.cpp b/ADApp/pluginTests/FFTPluginWrapper.cpp index 736882ec1..215954bc1 100644 --- a/ADApp/pluginTests/FFTPluginWrapper.cpp +++ b/ADApp/pluginTests/FFTPluginWrapper.cpp @@ -1,5 +1,5 @@ /* - * TimeSeriesPluginWrapper.cpp + * FFTPluginWrapper.cpp * * Created on: 21 Mar 2016 * Author: Ulrik Pedersen diff --git a/ADApp/pluginTests/FFTPluginWrapper.h b/ADApp/pluginTests/FFTPluginWrapper.h index c1bf9512e..fc61a0a1a 100644 --- a/ADApp/pluginTests/FFTPluginWrapper.h +++ b/ADApp/pluginTests/FFTPluginWrapper.h @@ -1,5 +1,5 @@ /* - * TimeSeriesPluginWrapper.cpp + * FFTPluginWrapper.cpp * * Created on: 21 Mar 2016 * Author: Ulrik Pedersen diff --git a/ADApp/pluginTests/Makefile b/ADApp/pluginTests/Makefile index 9b02287e9..44553717b 100644 --- a/ADApp/pluginTests/Makefile +++ b/ADApp/pluginTests/Makefile @@ -39,6 +39,7 @@ ifeq ($(WITH_BOOST),YES) ADTestUtility_SRCS += AttrPlotPluginWrapper.cpp ADTestUtility_SRCS += ROIPluginWrapper.cpp ADTestUtility_SRCS += OverlayPluginWrapper.cpp + ADTestUtility_SRCS += CodecPluginWrapper.cpp PROD_IOC_Linux += plugin-test PROD_IOC_Darwin += plugin-test @@ -53,6 +54,7 @@ ifeq ($(WITH_BOOST),YES) plugin-test_SRCS += test_NDPosPlugin.cpp plugin-test_SRCS += test_NDPluginTimeSeries.cpp plugin-test_SRCS += test_NDPluginFFT.cpp + plugin-test_SRCS += test_NDPluginCodec.cpp plugin-test_SRCS += test_NDPluginAttrPlot.cpp plugin-test_SRCS += test_NDPluginROI.cpp plugin-test_SRCS += test_NDPluginOverlay.cpp @@ -68,7 +70,7 @@ ifeq ($(WITH_BOOST),YES) boost_unit_test_framework_DIR=$(BOOST_LIB) plugin-test_LIBS_Linux += boost_unit_test_framework plugin-test_LIBS_Darwin += boost_unit_test_framework - USR_LDFLAGS_WIN32 += /LIBPATH:$(BOOST_LIB) + USR_LDFLAGS_WIN32 += /LIBPATH:$(BOOST_LIB) LIB_LIBS_WIN32 += libboost_unit_test_framework-vc141-mt-s-x64-1_69 else plugin-test_SYS_LIBS += boost_unit_test_framework @@ -76,7 +78,30 @@ ifeq ($(WITH_BOOST),YES) # Link order matters when doing a static build plugin-test_LIBS += ADTestUtility - + ifeq ($(WITH_JPEG),YES) + USR_CXXFLAGS += -DHAVE_JPEG + ifdef JPEG_INCLUDE + USR_INCLUDES += $(addprefix -I, $(JPEG_INCLUDE)) + endif + endif + ifeq ($(WITH_BLOSC), YES) + USR_CXXFLAGS += -DHAVE_BLOSC + endif + ifeq ($(WITH_BITSHUFFLE), YES) + USR_CXXFLAGS += -DHAVE_BITSHUFFLE + endif + ifeq ($(WITH_ZLIB), YES) + USR_CXXFLAGS += -DHAVE_ZLIB + endif + ifdef BLOSC_INCLUDE + USR_INCLUDES += $(addprefix -I, $(BLOSC_INCLUDE)) + endif + ifdef BITSHUFFLE_INCLUDE + USR_INCLUDES += $(addprefix -I, $(BITSHUFFLE_INCLUDE)) + endif + ifdef ZLIB_INCLUDE + USR_INCLUDES += $(addprefix -I, $(ZLIB_INCLUDE)) + endif ifdef HDF5_INCLUDE USR_INCLUDES += $(addprefix -I, $(HDF5_INCLUDE)) endif @@ -90,10 +115,10 @@ ifeq ($(WITH_BOOST),YES) USR_INCLUDES += $(addprefix -I, $(BOOST_INCLUDE)) endif ifeq ($(BOOST_USE_STATIC_LINK),YES) - USR_CXXFLAGS_Linux += -DBOOST_USE_STATIC_LINK - USR_CFLAGS_Linux += -DBOOST_USE_STATIC_LINK - USR_CXXFLAGS_WIN32 += -DBOOST_USE_STATIC_LINK - USR_CFLAGS_WIN32 += -DBOOST_USE_STATIC_LINK + USR_CXXFLAGS_Linux += -DBOOST_USE_STATIC_LINK + USR_CFLAGS_Linux += -DBOOST_USE_STATIC_LINK + USR_CXXFLAGS_WIN32 += -DBOOST_USE_STATIC_LINK + USR_CFLAGS_WIN32 += -DBOOST_USE_STATIC_LINK endif endif diff --git a/ADApp/pluginTests/test_NDPluginCodec.cpp b/ADApp/pluginTests/test_NDPluginCodec.cpp new file mode 100644 index 000000000..6981812ad --- /dev/null +++ b/ADApp/pluginTests/test_NDPluginCodec.cpp @@ -0,0 +1,407 @@ +/* + * test_NDPluginCodec.cpp + * + * Tests NDPluginCodec with compress/decompress for each codec. + * + * Created on: 20 Apr 2026 + * Author: Jakub Wlodek + */ + +#include +#include + +#include "boost/test/unit_test.hpp" + +#include +#include +#include +#include +#include + +#include "testingutilities.h" +#include "CodecPluginWrapper.h" + +/* Dimensions of the test array: 8x8 of UInt16 */ +#define TEST_XSIZE 8 +#define TEST_YSIZE 8 +#define TEST_NELEMENTS (TEST_XSIZE * TEST_YSIZE) + +struct CodecTestFixture +{ + NDArrayPool *arrayPool; + asynNDArrayDriver *dummy_driver; + CodecPluginWrapper *codec; + TestingPlugin *ds; + + CodecTestFixture() + { + std::string dummy_port("simCodec"), testport("testCodec"); + uniqueAsynPortName(dummy_port); + uniqueAsynPortName(testport); + + dummy_driver = new asynNDArrayDriver( + dummy_port.c_str(), 1, 0, 0, + asynGenericPointerMask, asynGenericPointerMask, 0, 0, 0, 0); + arrayPool = dummy_driver->pNDArrayPool; + + codec = new CodecPluginWrapper(testport.c_str(), dummy_port.c_str()); + codec->start(); + + ds = new TestingPlugin(testport.c_str(), 0); + + codec->write(NDPluginDriverEnableCallbacksString, 1); + codec->write(NDPluginDriverBlockingCallbacksString, 1); + } + + ~CodecTestFixture() + { + delete codec; + delete dummy_driver; + } + + /* Create a test array filled with known data */ + NDArray *createTestArray() + { + size_t dims[2] = {TEST_XSIZE, TEST_YSIZE}; + NDArray *arr = arrayPool->alloc(2, dims, NDUInt16, 0, NULL); + epicsUInt16 *pData = (epicsUInt16 *)arr->pData; + for (int i = 0; i < TEST_NELEMENTS; i++) { + pData[i] = (epicsUInt16)(i % 256); + } + return arr; + } + + /* Process an array through the plugin */ + void processArray(NDArray *pArray) + { + codec->lock(); + codec->processCallbacks(pArray); + codec->unlock(); + } + + /* Verify that decompressed data matches the original */ + void verifyDataMatch(NDArray *original, NDArray *result) + { + BOOST_REQUIRE(original->pData != NULL); + BOOST_REQUIRE(result->pData != NULL); + NDArrayInfo_t origInfo, resultInfo; + original->getInfo(&origInfo); + result->getInfo(&resultInfo); + BOOST_REQUIRE_EQUAL(origInfo.totalBytes, resultInfo.totalBytes); + BOOST_REQUIRE_EQUAL(result->dataType, original->dataType); + BOOST_CHECK_EQUAL(memcmp(original->pData, result->pData, origInfo.totalBytes), 0); + } +}; + +BOOST_FIXTURE_TEST_SUITE(CodecPluginTests, CodecTestFixture) + +/* Test that passthrough with no codec works */ +BOOST_AUTO_TEST_CASE(test_no_compression) +{ + NDArray *input = createTestArray(); + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_NONE); + + ds->arrays.clear(); + processArray(input); + + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + NDArray *output = ds->arrays[0]; + BOOST_CHECK(output->codec.empty()); + verifyDataMatch(input, output); + input->release(); +} + +#ifdef HAVE_JPEG +BOOST_AUTO_TEST_CASE(test_jpeg_compress_decompress) +{ + /* JPEG is lossy so we only check that the compress/decompress produces a valid + array of the same dimensions, not bit-identical data. + JPEG only supports 8-bit data. */ + size_t dims[2] = {TEST_XSIZE, TEST_YSIZE}; + NDArray *input = arrayPool->alloc(2, dims, NDUInt8, 0, NULL); + epicsUInt8 *pData = (epicsUInt8 *)input->pData; + for (int i = 0; i < TEST_NELEMENTS; i++) { + pData[i] = (epicsUInt8)(i % 256); + } + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_JPEG); + codec->write(NDCodecJPEGQualityString, 95); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_JPEG]); + BOOST_CHECK(compressed->compressedSize > 0); + BOOST_CHECK(compressed->compressedSize <= compressed->dataSize); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + BOOST_CHECK_EQUAL(decompressed->dataType, NDUInt8); + + NDArrayInfo_t info; + decompressed->getInfo(&info); + BOOST_CHECK(info.totalBytes > 0); + + input->release(); +} +#endif /* HAVE_JPEG */ + +#ifdef HAVE_ZLIB +BOOST_AUTO_TEST_CASE(test_zlib_compress_decompress) +{ + NDArray *input = createTestArray(); + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_ZLIB); + codec->write(NDCodecZlibCLevelString, 6); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_ZLIB]); + BOOST_CHECK(compressed->compressedSize > 0); + BOOST_CHECK(compressed->compressedSize <= compressed->dataSize); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + verifyDataMatch(input, decompressed); + + input->release(); +} + +BOOST_AUTO_TEST_CASE(test_zlib_compression_levels) +{ + /* Test that different compression levels all produce valid compress/decompress */ + for (int level = 0; level <= 9; level += 3) { + NDArray *input = createTestArray(); + + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_ZLIB); + codec->write(NDCodecZlibCLevelString, level); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_ZLIB]); + + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + verifyDataMatch(input, decompressed); + + input->release(); + } +} +#endif /* HAVE_ZLIB */ + +#ifdef HAVE_BLOSC +BOOST_AUTO_TEST_CASE(test_blosc_compress_decompress) +{ + NDArray *input = createTestArray(); + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_BLOSC); + codec->write(NDCodecBloscCLevelString, 5); + codec->write(NDCodecBloscShuffleString, 1); + codec->write(NDCodecBloscCompressorString, NDCODEC_BLOSC_BLOSCLZ); + codec->write(NDCodecBloscNumThreadsString, 1); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_BLOSC]); + BOOST_CHECK(compressed->compressedSize > 0); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + verifyDataMatch(input, decompressed); + + input->release(); +} + +BOOST_AUTO_TEST_CASE(test_blosc_compressors) +{ + /* Test several blosc sub-compressors compress/decompress. */ + int compressors[] = { + NDCODEC_BLOSC_BLOSCLZ, + NDCODEC_BLOSC_LZ4, + NDCODEC_BLOSC_LZ4HC, + NDCODEC_BLOSC_ZLIB, + }; + int nCompressors = sizeof(compressors) / sizeof(compressors[0]); + + for (int c = 0; c < nCompressors; c++) { + NDArray *input = createTestArray(); + + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_BLOSC); + codec->write(NDCodecBloscCLevelString, 5); + codec->write(NDCodecBloscShuffleString, 1); + codec->write(NDCodecBloscCompressorString, compressors[c]); + codec->write(NDCodecBloscNumThreadsString, 1); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_BLOSC]); + + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + verifyDataMatch(input, decompressed); + + input->release(); + } +} +#endif /* HAVE_BLOSC */ + +#ifdef HAVE_BITSHUFFLE +BOOST_AUTO_TEST_CASE(test_lz4_compress_decompress) +{ + NDArray *input = createTestArray(); + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_LZ4); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_LZ4]); + BOOST_CHECK(compressed->compressedSize > 0); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + verifyDataMatch(input, decompressed); + + input->release(); +} + +BOOST_AUTO_TEST_CASE(test_lz4hdf5_compress_decompress) +{ + NDArray *input = createTestArray(); + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_LZ4HDF5); + codec->write(NDCodecLZ4HDF5BlockSizeString, 0); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_LZ4HDF5]); + BOOST_CHECK(compressed->compressedSize > 0); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + verifyDataMatch(input, decompressed); + + input->release(); +} + +BOOST_AUTO_TEST_CASE(test_bslz4_compress_decompress) +{ + NDArray *input = createTestArray(); + + /* Compress */ + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_BSLZ4); + + ds->arrays.clear(); + processArray(input); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *compressed = ds->arrays[0]; + BOOST_CHECK_EQUAL(compressed->codec.name, codecName[NDCODEC_BSLZ4]); + BOOST_CHECK(compressed->compressedSize > 0); + + /* Decompress */ + codec->write(NDCodecModeString, NDCODEC_DECOMPRESS); + ds->arrays.clear(); + processArray(compressed); + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + + NDArray *decompressed = ds->arrays[0]; + BOOST_CHECK(decompressed->codec.empty()); + verifyDataMatch(input, decompressed); + + input->release(); +} +#endif /* HAVE_BITSHUFFLE */ + +/* Test that compressing an already-compressed array is handled gracefully */ +BOOST_AUTO_TEST_CASE(test_double_compress_warning) +{ + NDArray *input = createTestArray(); + input->codec.name = codecName[NDCODEC_ZLIB]; + input->compressedSize = 10; + + codec->write(NDCodecModeString, NDCODEC_COMPRESS); + codec->write(NDCodecCompressorString, NDCODEC_ZLIB); + + ds->arrays.clear(); + processArray(input); + /* Plugin should still produce output (the original array passed through) */ + BOOST_REQUIRE_EQUAL(ds->arrays.size(), (size_t)1); + /* Status should indicate warning */ + BOOST_CHECK_EQUAL(codec->readInt(NDCodecCodecStatusString), (int)NDCODEC_WARNING); + + input->release(); +} + +BOOST_AUTO_TEST_SUITE_END()