From 456c582c1891b2a970c67589649557d4172f5f60 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 15 Oct 2019 13:24:14 -0600 Subject: [PATCH 1/7] lf --- .../data/testsuites/hdf-testsuite/dumpeos.sh | 18 ++++++++--------- .../data/testsuites/hdf-testsuite/dumptest.sh | 20 +++++++++---------- opendap/testsuites/hdf-testsuite/dumpeos.sh | 18 ++++++++--------- opendap/testsuites/hdf-testsuite/dumptest.sh | 20 +++++++++---------- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/opendap/src/test/data/testsuites/hdf-testsuite/dumpeos.sh b/opendap/src/test/data/testsuites/hdf-testsuite/dumpeos.sh index a6dfa9fc5e..38475a7e90 100644 --- a/opendap/src/test/data/testsuites/hdf-testsuite/dumpeos.sh +++ b/opendap/src/test/data/testsuites/hdf-testsuite/dumpeos.sh @@ -1,9 +1,9 @@ -#!/bin/sh - -for i in PointFile GridFile SwathFile -do -echo ${i} -../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.das -../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.dds -../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/${i}.hdf?" > ${i}.data -done +#!/bin/sh + +for i in PointFile GridFile SwathFile +do +echo ${i} +../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.das +../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.dds +../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/${i}.hdf?" > ${i}.data +done diff --git a/opendap/src/test/data/testsuites/hdf-testsuite/dumptest.sh b/opendap/src/test/data/testsuites/hdf-testsuite/dumptest.sh index 711c5539fd..b4bb6515e9 100644 --- a/opendap/src/test/data/testsuites/hdf-testsuite/dumptest.sh +++ b/opendap/src/test/data/testsuites/hdf-testsuite/dumptest.sh @@ -1,10 +1,10 @@ -#!/bin/sh - -#for i in an1 dfp1 dfr1 gr1 gr2 gr3 gr4 gr5 sds1 sds2 sds3 sds4 sds5 sds6 sds7 vs1 vs2 vs3 vs4 vs5 -for i in gr4 -do -echo test${i} -../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.das -../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.dds -../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/test${i}.hdf?" > test${i}.data -done +#!/bin/sh + +#for i in an1 dfp1 dfr1 gr1 gr2 gr3 gr4 gr5 sds1 sds2 sds3 sds4 sds5 sds6 sds7 vs1 vs2 vs3 vs4 vs5 +for i in gr4 +do +echo test${i} +../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.das +../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.dds +../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/test${i}.hdf?" > test${i}.data +done diff --git a/opendap/testsuites/hdf-testsuite/dumpeos.sh b/opendap/testsuites/hdf-testsuite/dumpeos.sh index a6dfa9fc5e..38475a7e90 100644 --- a/opendap/testsuites/hdf-testsuite/dumpeos.sh +++ b/opendap/testsuites/hdf-testsuite/dumpeos.sh @@ -1,9 +1,9 @@ -#!/bin/sh - -for i in PointFile GridFile SwathFile -do -echo ${i} -../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.das -../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.dds -../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/${i}.hdf?" > ${i}.data -done +#!/bin/sh + +for i in PointFile GridFile SwathFile +do +echo ${i} +../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.das +../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/${i}.hdf" > ${i}.dds +../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/${i}.hdf?" > ${i}.data +done diff --git a/opendap/testsuites/hdf-testsuite/dumptest.sh b/opendap/testsuites/hdf-testsuite/dumptest.sh index 711c5539fd..b4bb6515e9 100644 --- a/opendap/testsuites/hdf-testsuite/dumptest.sh +++ b/opendap/testsuites/hdf-testsuite/dumptest.sh @@ -1,10 +1,10 @@ -#!/bin/sh - -#for i in an1 dfp1 dfr1 gr1 gr2 gr3 gr4 gr5 sds1 sds2 sds3 sds4 sds5 sds6 sds7 vs1 vs2 vs3 vs4 vs5 -for i in gr4 -do -echo test${i} -../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.das -../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.dds -../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/test${i}.hdf?" > test${i}.data -done +#!/bin/sh + +#for i in an1 dfp1 dfr1 gr1 gr2 gr3 gr4 gr5 sds1 sds2 sds3 sds4 sds5 sds6 sds7 vs1 vs2 vs3 vs4 vs5 +for i in gr4 +do +echo test${i} +../../../bin/geturl -a "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.das +../../../bin/geturl -d "http://manta/cgi-bin/nph-hdf/test${i}.hdf" > test${i}.dds +../../../bin/geturl -D "http://manta/cgi-bin/nph-hdf/test${i}.hdf?" > test${i}.data +done From 773baf7564a97dfa0d246ab898f36b9126e247a8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 15 Oct 2019 13:25:34 -0600 Subject: [PATCH 2/7] Fix DAP4 code to support its use with e.g. .ncml files The old DAP4 code assumed that all source files on the server could be specified using a path string. It turns out that for virtual files like .ncml files, this is false and instead, a NetcdfFile instance must be used. Fixing this necessitated some significant changes to the DAP4 code. The basic fix was to make the primary DAP4 code operate with respect to a DSP object. Then the problem was to change the code that invoked DAP4 so that it figured out what kind of DSP object to use. The primary change on the server side is to provide a function that, given an object, can figure out what kind of DSP to use to convert that kind of object to a DAP4 representation. This is defined in the new class called DapDSP. As a rule, clients will end up using HttpDSP to read the incoming DAP4 encode Http stream. Again as a rule, Servers will assume an incoming Http request is for a NetcdfFile and will use CDMDSP to convert that to a DAP4 Http stream. In the event that Server detects that the file to be read is .nc, then it may choose to use Nc4DSP to read the file, although it is not required to and can instead open the file as a NetcdfFile object and use CDMDSP. The other major change in this PR, and the one that actually prompted this whole change, is in the DapController and its subclasses. Specifically, the function getNetcdfFile() has been added to attempt to convert a String (a location) to a NetcdfFile object (ultimately via DapDSP). It is only if this fails that an attempt is made to look for the other cases via the location extension. Misc. Changes: - Remove DSPRegistry as no longer needed. - Remove ThreddsDSP since it was just a wrapper around CDMDSP. - Add a simple .ncml test case to show that NCML files could be served. - Use #mode= to determine if DAP4 or DAP2 is to be used instead of #protocol= or #proto=. The latter two are still supported for back compatibility, but are no longer documented and are deprecated. --- cdm/src/main/java/ucar/nc2/NetcdfFile.java | 2 +- .../java/ucar/nc2/dataset/DatasetUrl.java | 22 +- dap4/build.gradle | 4 +- .../src/main/java/dap4/cdm/dsp/CDMDSP.java | 44 +-- .../src/main/java/dap4/cdm/nc2/CDMArray.java | 1 - .../java/dap4/cdm/nc2/CDMArrayAtomic.java | 8 - .../java/dap4/cdm/nc2/CDMArrayDelegate.java | 9 - .../java/dap4/cdm/nc2/CDMArraySequence.java | 8 - .../java/dap4/cdm/nc2/CDMArrayStructure.java | 8 - .../main/java/dap4/cdm/nc2/DapNetcdfFile.java | 24 +- .../src/main/java/dap4/core/data/DSP.java | 85 +++-- .../main/java/dap4/core/data/DSPRegistry.java | 177 ---------- .../main/java/dap4/core/data/DataCursor.java | 2 - .../java/dap4/dap4lib/AbstractCursor.java | 1 - .../main/java/dap4/dap4lib/AbstractDSP.java | 38 +-- .../src/main/java/dap4/dap4lib/FileDSP.java | 20 +- .../src/main/java/dap4/dap4lib/HttpDSP.java | 15 +- .../main/java/dap4/dap4lib/netcdf/Nc4DSP.java | 33 +- dap4/d4servlet/build.gradle | 1 + .../src/main/java/dap4/servlet/DapCache.java | 71 ++-- .../main/java/dap4/servlet/DapController.java | 35 +- .../src/main/java/dap4/servlet/DapDSP.java | 315 ++++++++++++++++++ .../main/java/dap4/servlet/DapRequest.java | 11 +- .../src/main/java/dap4/servlet/SynDSP.java | 22 +- .../TestServlet/baseline/test_ncml.ncml.dmr | 16 + .../data/resources/testfiles/test_ncml.ncml | 9 + .../test/java/dap4/test/D4TSController.java | 43 ++- .../test/java/dap4/test/DapTestCommon.java | 3 - .../test/java/dap4/test/TestCDMClient.java | 4 +- .../src/test/java/dap4/test/TestDSP.java | 27 +- .../src/test/java/dap4/test/TestDSR.java | 3 - .../test/java/dap4/test/TestFrontPage.java | 3 - .../src/test/java/dap4/test/TestSerial.java | 16 - .../src/test/java/dap4/test/TestServlet.java | 30 +- .../dap4/test/TestServletConstraints.java | 1 - .../src/main/java/dap4/d4ts/D4TSServlet.java | 62 ++-- .../thredds/server/dap4/Dap4Controller.java | 101 +++--- .../java/thredds/server/dap4/ThreddsDSP.java | 73 ---- .../unidata/util/test/UnitTestCommon.java | 1 + 39 files changed, 714 insertions(+), 634 deletions(-) delete mode 100644 dap4/d4core/src/main/java/dap4/core/data/DSPRegistry.java create mode 100644 dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java create mode 100644 dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dmr create mode 100644 dap4/d4tests/src/test/data/resources/testfiles/test_ncml.ncml delete mode 100644 tds/src/main/java/thredds/server/dap4/ThreddsDSP.java diff --git a/cdm/src/main/java/ucar/nc2/NetcdfFile.java b/cdm/src/main/java/ucar/nc2/NetcdfFile.java index 4aec293e18..2096c0ad74 100644 --- a/cdm/src/main/java/ucar/nc2/NetcdfFile.java +++ b/cdm/src/main/java/ucar/nc2/NetcdfFile.java @@ -635,7 +635,7 @@ static private ucar.unidata.io.RandomAccessFile getRaf(String location, int buff log.warn("Failed to uncompress {}, err= {}; try as a regular file.", uriString, e.getMessage()); //allow to fall through to open the "compressed" file directly - may be a misnamed suffix } finally { - stringLocker.release(uriString); + stringLocker.release(uriString); } if (uncompressedFileName != null) { diff --git a/cdm/src/main/java/ucar/nc2/dataset/DatasetUrl.java b/cdm/src/main/java/ucar/nc2/dataset/DatasetUrl.java index 91fb8cecca..6a5b497399 100644 --- a/cdm/src/main/java/ucar/nc2/dataset/DatasetUrl.java +++ b/cdm/src/main/java/ucar/nc2/dataset/DatasetUrl.java @@ -111,7 +111,7 @@ static public DatasetUrl findDatasetUrl(String orgLocation) throws IOException { // Priority in deciding // the service type is as follows. - // 1. "protocol" tag in fragment + // 1. "mode" tag in fragment (or "proto" or "protocol" for back compatibility) // 2. specific protocol in fragment // 3. leading protocol // 4. path extension @@ -196,22 +196,24 @@ static private ServiceType searchFragment(String fragment) { return null; Map map = parseFragment(fragment); if (map == null) return null; - String protocol = map.get("protocol"); - if(protocol == null) { + String mode = map.get("mode"); + if(mode == null) mode = map.get("proto"); + if(mode == null) mode = map.get("protocol"); + if(mode == null) { for(String p: FRAGPROTOCOLS) { - if(map.get(p) != null) {protocol = p; break;} + if(map.get(p) != null) {mode = p; break;} } } - if (protocol != null) { - if (protocol.equalsIgnoreCase("dap") || protocol.equalsIgnoreCase("dods")) + if (mode != null) { + if (mode.equalsIgnoreCase("dap") || mode.equalsIgnoreCase("dods")) return ServiceType.OPENDAP; - if (protocol.equalsIgnoreCase("dap4")) + if (mode.equalsIgnoreCase("dap4")) return ServiceType.DAP4; - if (protocol.equalsIgnoreCase("cdmremote")) + if (mode.equalsIgnoreCase("cdmremote")) return ServiceType.CdmRemote; - if (protocol.equalsIgnoreCase("thredds")) + if (mode.equalsIgnoreCase("thredds")) return ServiceType.THREDDS; - if (protocol.equalsIgnoreCase("ncml")) + if (mode.equalsIgnoreCase("ncml")) return ServiceType.NCML; } return null; diff --git a/dap4/build.gradle b/dap4/build.gradle index fec9a2ca03..eac6e71542 100644 --- a/dap4/build.gradle +++ b/dap4/build.gradle @@ -1,5 +1,7 @@ ext.title = "Data Access Protocol (DAP) version 4.0" // Will be inherited by subprojects. +// WARNING: this is no long accurate + // dap4 has no source code nor any artifacts to publish // NON-TRANSITIVE Dependency graph: @@ -10,7 +12,7 @@ ext.title = "Data Access Protocol (DAP) version 4.0" // Will be inherited by su // d4lib d4core httpservices // d4cdm d4lib cdm // server: -// d4servletshared d4cdmshared netcdf4 +// d4servlet d4cdm d4lib netcdf4 // d4ts d4servletshared // d4tswar d4ts // tests: diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/dsp/CDMDSP.java b/dap4/d4cdm/src/main/java/dap4/cdm/dsp/CDMDSP.java index e020013409..56524ab83a 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/dsp/CDMDSP.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/dsp/CDMDSP.java @@ -86,39 +86,27 @@ public CDMDSP() { } - public CDMDSP(String path) - throws DapException - { - super(); - setLocation(path); - } - ////////////////////////////////////////////////// // DSP Interface - // This is intended to be the last DSP checked - @Override - public boolean dspMatch(String path, DapContext context) - { - return true; - } + public String getLocation() {return this.ncdfile.getLocation();} /** - * @param filepath - absolute path to a file + * @param ncfile - source netcdf file * @return CDMDSP instance * @throws DapException */ @Override public CDMDSP - open(String filepath) + open(NetcdfFile ncfile) throws DapException { try { - NetcdfFile ncfile = createNetcdfFile(filepath, null); NetcdfDataset ncd = new NetcdfDataset(ncfile, ENHANCEMENT); + this.ncdfile = ncd; return open(ncd); } catch (IOException ioe) { - throw new DapException("CDMDSP: cannot process: " + filepath, ioe); + throw new DapException("CDMDSP: cannot process: " + getLocation(), ioe); } } @@ -132,10 +120,8 @@ public boolean dspMatch(String path, DapContext context) public CDMDSP open(NetcdfDataset ncd) throws DapException { - assert this.context != null; this.dmrfactory = new DMRFactory(); this.ncdfile = ncd; - setLocation(this.ncdfile.getLocation()); buildDMR(); return this; } @@ -1071,26 +1057,6 @@ else if(atype == TypeSort.Float64) return o; } */ - ////////////////////////////////////////////////// - - protected NetcdfFile - createNetcdfFile(String location, CancelTask canceltask) - throws DapException - { - try { - NetcdfFile ncfile = NetcdfFile.open(location, -1, canceltask, getContext()); - return ncfile; - } catch (DapException de) { - if (DEBUG) - de.printStackTrace(); - throw de; - } catch (Exception e) { - if (DEBUG) - e.printStackTrace(); - throw new DapException(e); - } - } - ////////////////////////////////////////////////// // Utilities diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArray.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArray.java index 435e0632d3..d9a3414b1e 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArray.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArray.java @@ -15,7 +15,6 @@ /*package*/ interface CDMArray { - public DSP getDSP(); public DapVariable getTemplate(); public long getSizeBytes(); // In bytes public DapType getBaseType(); diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayAtomic.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayAtomic.java index bc74015604..5e544cf6b4 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayAtomic.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayAtomic.java @@ -40,7 +40,6 @@ ///////////////////////////////////////////////////// // Instance variables - protected DSP dsp = null; protected DapVariable template = null; protected DapType basetype = null; @@ -64,7 +63,6 @@ { super(CDMTypeFcns.daptype2cdmtype(((DapVariable) data.getTemplate()).getBaseType()), CDMUtil.computeEffectiveShape(((DapVariable) data.getTemplate()).getDimensions())); - this.dsp = data.getDSP(); this.data = data; this.template = (DapVariable) this.data.getTemplate(); this.basetype = this.template.getBaseType(); @@ -83,12 +81,6 @@ return this.basetype; } - @Override - public DSP getDSP() - { - return this.dsp; - } - @Override public DapVariable getTemplate() { diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayDelegate.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayDelegate.java index 96adfdb79e..91bb0687ff 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayDelegate.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayDelegate.java @@ -6,7 +6,6 @@ import dap4.core.dmr.*; import dap4.core.data.DSP; -import dap4.core.data.DSP; /** * Since we can't use AbstractCDMArray, @@ -20,7 +19,6 @@ /*package*/ class CDMArrayDelegate implements CDMArray { - protected DSP dsp = null; protected DapVariable template = null; protected long bytesize = 0; protected DapType basetype = null; @@ -31,18 +29,11 @@ this.template = template; this.bytesize = 0; this.root = root; - this.dsp = dsp; this.bytesize = size; this.basetype = this.template.getBaseType(); this.primitivetype = this.basetype.getPrimitiveType(); */ - @Override - public DSP getDSP() - { - return this.dsp; - } - @Override public DapVariable getTemplate() { diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArraySequence.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArraySequence.java index e0ed4a0b94..1e6709ee75 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArraySequence.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArraySequence.java @@ -101,7 +101,6 @@ public int getCurrentRecno() // Instance variables protected Group cdmroot = null; - protected DSP dsp; protected DapVariable template; protected DapType basetype; protected long bytesize = 0; @@ -143,7 +142,6 @@ public int getCurrentRecno() throw new DapException("Non-scalar sequences unsupported through CDM interface"); assert data.getScheme() == DataCursor.Scheme.SEQARRAY; this.cdmroot = group; - this.dsp = dsp; // Since this is a scalar, pull out the single instance this.seqdata = ((DataCursor[])data.read(dap4.core.util.Index.SCALAR))[0]; this.recordcount = this.seqdata.getRecordCount(); @@ -189,12 +187,6 @@ public DapType getBaseType() return this.basetype; } - @Override - public DSP getDSP() - { - return this.dsp; - } - @Override public DapVariable getTemplate() { diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayStructure.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayStructure.java index 070ccc602c..7f0f3d2358 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayStructure.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/CDMArrayStructure.java @@ -65,7 +65,6 @@ static protected class FieldArrays // CDMArry variables protected Group cdmroot = null; - protected DSP dsp = null; protected DapVariable template = null; protected DapType basetype = null; protected long dimsize = 0; @@ -104,7 +103,6 @@ static protected class FieldArrays CDMUtil.computeEffectiveShape(((DapVariable) data.getTemplate()).getDimensions())); this.template = (DapVariable) data.getTemplate(); assert data.getScheme() == Scheme.STRUCTARRAY; - this.dsp = data.getDSP(); this.cdmroot = cdmroot; this.basetype = this.template.getBaseType(); this.dimsize = DapUtil.dimProduct(template.getDimensions()); @@ -139,12 +137,6 @@ static protected class FieldArrays ////////////////////////////////////////////////// // CDMArray Interface - @Override - public DSP getDSP() - { - return this.dsp; - } - @Override public DapVariable getTemplate() { diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java index cac6e046bd..a03f378ff7 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java @@ -7,7 +7,6 @@ import dap4.cdm.CDMUtil; import dap4.cdm.NodeMap; import dap4.core.data.DSP; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapContext; import dap4.core.util.DapException; import dap4.core.util.DapUtil; @@ -23,7 +22,9 @@ import ucar.nc2.iosp.IospHelper; import ucar.nc2.util.CancelTask; +import java.io.File; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.nio.channels.WritableByteChannel; import java.util.*; @@ -67,16 +68,6 @@ public void setProgress(String msg, int progress) static protected final NullCancelTask nullcancel = new NullCancelTask(); - /** - * Define a map of known DSP classes. - */ - static protected DSPRegistry dspregistry = new DSPRegistry(); - - static { - dspregistry.register(FileDSP.class, DSPRegistry.FIRST); - dspregistry.register(HttpDSP.class, DSPRegistry.FIRST); - } - ////////////////////////////////////////////////// // Instance Variables @@ -125,19 +116,22 @@ public DapNetcdfFile(String location, CancelTask cancelTask) throw new IOException(use); } boolean isfile = xuri.isFile(); + DapContext cxt = new DapContext(); if(isfile) { this.dsplocation = DapUtil.absolutize(xuri.getPath()); + this.dsp = new FileDSP().open(new File(this.dsplocation)); } else { // Not a file url this.dsplocation = xuri.assemble(XURI.URLBASE); + try { + this.dsp = new FileDSP().open(new URI(this.dsplocation)); + } catch (URISyntaxException e) { + throw new DapException(e); + } } - DapContext cxt = new DapContext(); cancel = (cancelTask == null ? nullcancel : cancelTask); - // 1. Get and parse the constrained DMR and Data v-a-v URL - this.dsp = dspregistry.findMatchingDSP(location,cxt); // will set dsp context if(this.dsp == null) throw new IOException("No matching DSP: "+this.location); this.dsp.setContext(cxt); - this.dsp.open(this.dsplocation); // 2. Construct an equivalent CDM tree and populate // this NetcdfFile object. diff --git a/dap4/d4core/src/main/java/dap4/core/data/DSP.java b/dap4/d4core/src/main/java/dap4/core/data/DSP.java index 42063a0082..a009fd8517 100644 --- a/dap4/d4core/src/main/java/dap4/core/data/DSP.java +++ b/dap4/d4core/src/main/java/dap4/core/data/DSP.java @@ -15,41 +15,53 @@ import java.util.List; import java.util.Map; +/* +The DAP4 code operates with respect to a DSP object given to it +as part of the servlet process. The DSP subclass to use is found +by a function that, given an object, can figure out what kind of +DSP to use to convert that kind of object to a DAP4 +representation. This is defined in the new class called +DapDSP. DapDSP has two primary externally visible API functions: +1. open(...,NetcdfFile nc,...) - returns a CDMDSP wrapping the + NetcdfFile object. +2. open(...,String path,...) - uses the rules defined in DapDSP + to figure out what kind of DSP to return. + +The second open() is used to access specialized data sources +other than those that can be represented as NetcdfFile objects. +In particular, it is possible to build a DSP that wraps a stream +or file containing direct DAP4 protocol encoded information, as +opposed to CDM structured information via NetcdfFile. + +The current set of cases is as follows: + +1. Data sources representable as NetcdfFile objects are used + by CDMDSP on servers to access the object structured as + CDM. The DSP then provides an API that the DAP4 code can then + convert to the DAP4 protocol and send out to clients. + +2. DAP4 encoded data sources are used by client programs and + test programs to interpret a DAP4 protocol stream (or file if + used for testing). This stream is then made available on + clients in the form of CDM. + +So we have these cases: + +1. NetcdfFile -> CDMDSP -> DAP4 (on the server) +2. DAP4 -> HttpDSP -> CDM (on the client) +3. DAP4 test file data -> DSP (i.e. FileDSP|SynDSP) -> DAP4 + +The last case is specifically for testing on the server or +client, so cases 1 and 2 are the critical ones. + +The bottom line is that the DSP interface exposes no ability to +create a DSP, although some parametric setting is allowed. + +The actual construction of a DSP instance is handled by DapDSP. +*/ + public interface DSP { - /* All implementing classes must implement: - 1. a static dspMatch() function - 2. A parameterless constructor - */ - - /** - * Determine if a path refers to an object processable by this DSP - * - * @param path - * @param context - * @return true if this path can be processed by an instance of this DSP - */ - public boolean dspMatch(String path, DapContext context); - - /** - * @param path It is assumed that the path - * is appropriate to the dsp - * E.g. an absolute path or a url - * @return DSP wrapping the path source - * @throws dap4.core.util.DapException - */ - public DSP open(String path) throws DapException; - - public void close() throws IOException; - - public String getLocation(); - - public DSP setLocation(String location); - - public Object getContext(); - - public void setContext(DapContext cxt); - public DapDataset getDMR() throws dap4.core.util.DapException; public DataCursor getVariableData(DapVariable var) throws DapException; @@ -58,4 +70,11 @@ public interface DSP public ChecksumMode getChecksumMode(); + public void close() throws IOException; + + public Object getContext(); + public void setContext(DapContext cxt); + + public String getLocation(); + } diff --git a/dap4/d4core/src/main/java/dap4/core/data/DSPRegistry.java b/dap4/d4core/src/main/java/dap4/core/data/DSPRegistry.java deleted file mode 100644 index 937fb5ff09..0000000000 --- a/dap4/d4core/src/main/java/dap4/core/data/DSPRegistry.java +++ /dev/null @@ -1,177 +0,0 @@ -/* Copyright 2012, UCAR/Unidata. - See the LICENSE file for more information. */ - -package dap4.core.data; - -import dap4.core.util.DapContext; -import dap4.core.util.DapException; - -import java.util.ArrayList; -import java.util.List; - -/** - */ - -public class DSPRegistry -{ - ////////////////////////////////////////////////// - // Constants - - // MNemonics - static public final boolean LAST = true; - static public final boolean FIRST = false; - - ////////////////////////////////////////////////// - // Type Decls - - static protected class Registration - { - Class dspclass; - DSP matcher; - - public Registration(Class cl) - { - this.dspclass = cl; - try { - this.matcher = dspclass.newInstance(); - } catch (IllegalAccessException | InstantiationException e) { - throw new IllegalArgumentException("DSPFactory: cannot create matching instance for Class: " + dspclass.getName()); - } - } - - public String toString() - { - return dspclass.getName(); - } - } - - ////////////////////////////////////////////////// - - static protected ClassLoader loader = DSPRegistry.class.getClassLoader(); - - ////////////////////////////////////////////////// - // Instance Variables - - /** - * Define a map of known DSP classes. - * Must be ordered to allow control over - * test order - */ - protected List registry = new ArrayList<>(); - - ////////////////////////////////////////////////// - // Constructor(s) - - public DSPRegistry() - { - } - - - ////////////////////////////////////////////////// - // Accessors - - static public void setLoader(ClassLoader ldr) - { - loader = ldr; - } - - ////////////////////////////////////////////////// - // API - - /** - * Register a DSP, using its class string name. - * - * @param className Name of class that implements DSP. - * @throws IllegalAccessException if class is not accessible. - * @throws InstantiationException if class doesnt have a no-arg constructor. - * @throws ClassNotFoundException if class not found. - */ - synchronized public void register(String className, boolean last) - throws DapException - { - try { - Class klass = (Class) loader.loadClass(className); - register(klass, last); - } catch (ClassNotFoundException e) { - throw new DapException(e); - } - } - - /** - * Register a DSP class. - * - * @param klass Class that implements DSP. - * @param last true=>insert at the end of the list; otherwise front - * @throws IllegalAccessException if class is not accessible. - * @throws InstantiationException if class doesnt have a no-arg constructor. - * @throws ClassCastException if class doesnt implement DSP interface. - */ - synchronized public void - register(Class klass, boolean last) - { - // is this already defined? - if(registered(klass)) return; - if(last) - registry.add(new Registration(klass)); - else - registry.add(0, new Registration(klass)); - } - - /** - * See if a specific DSP is registered - * - * @param klass Class for which to search - */ - - synchronized public boolean - registered(Class klass) - { - for(Registration r : registry) { - if(r.dspclass == klass) return true; - } - return false; - } - - /** - * Unregister dsp. - * - * @param klass Class for which to search - */ - synchronized public void - unregister(Class klass) - { - for(int i = 0; i < registry.size(); i++) { - if(registry.get(i).dspclass == klass) { - registry.remove(i); - break; - } - } - } - - /** - * @param path - * @return new DSP object that can process this path - * @throws DapException - */ - - synchronized public DSP - findMatchingDSP(String path, DapContext cxt) - throws DapException - { - for(int i = 0; i < registry.size(); i++) { - try { - Registration tester = registry.get(i); - boolean ismatch = (Boolean) tester.matcher.dspMatch(path, cxt); - if(ismatch) { - DSP dsp = (DSP) tester.dspclass.newInstance(); - return dsp; - } - } catch (Exception e) { - throw new DapException(e); - } - } - throw new IllegalArgumentException("Cannot open " + path); - } - -} - diff --git a/dap4/d4core/src/main/java/dap4/core/data/DataCursor.java b/dap4/d4core/src/main/java/dap4/core/data/DataCursor.java index eb3a0d6ea0..d20cf35b0b 100644 --- a/dap4/d4core/src/main/java/dap4/core/data/DataCursor.java +++ b/dap4/d4core/src/main/java/dap4/core/data/DataCursor.java @@ -48,8 +48,6 @@ public boolean isCompoundArray() public Scheme getScheme(); - public DSP getDSP(); - public DapNode getTemplate(); public Index getIndex() throws DapException; diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractCursor.java b/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractCursor.java index 496d6b2036..79afabe34d 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractCursor.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractCursor.java @@ -115,7 +115,6 @@ public Scheme getScheme() return this.scheme; } - @Override public DSP getDSP() { diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractDSP.java b/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractDSP.java index 3cb65bc339..3cf74ea144 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractDSP.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/AbstractDSP.java @@ -17,10 +17,14 @@ import dap4.core.util.DapException; import dap4.core.util.DapUtil; import org.xml.sax.SAXException; +import ucar.nc2.NetcdfFile; +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.URI; +import java.net.URL; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.HashMap; @@ -56,7 +60,6 @@ abstract public class AbstractDSP implements DSP protected DapContext context = null; protected DapDataset dmr = null; - protected String location = null; private ByteOrder order = null; private ChecksumMode checksummode = ChecksumMode.DAP; @@ -77,19 +80,27 @@ public AbstractDSP() /* must have a parameterless constructor */ /** * "open" a reference to a data source and return the DSP wrapper. + * There is one for each possible kind of source * - * @param location - Object that defines the data source - * @return = wrapping dsp - * @throws DapException + * @param src - Object that defines the data source + * @return wrapping dsp + * @throws DapException | UnsupportedOperationException */ - @Override - abstract public AbstractDSP open(String location) throws DapException; + public AbstractDSP open(NetcdfFile src) throws DapException + {throw new UnsupportedOperationException();} + public AbstractDSP open(File src) throws DapException + {throw new UnsupportedOperationException();} + public AbstractDSP open(URI src) throws DapException + {throw new UnsupportedOperationException();} /** * @throws IOException */ abstract public void close() throws IOException; + // Misc + abstract public String getLocation(); + ////////////////////////////////////////////////// // Implemented @@ -106,21 +117,6 @@ public DapContext getContext() return this.context; } - @Override - public String - getLocation() - { - return this.location; - } - - @Override - public AbstractDSP - setLocation(String loc) - { - this.location = loc; - return this; - } - @Override public DapDataset getDMR() { diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/FileDSP.java b/dap4/d4lib/src/main/java/dap4/dap4lib/FileDSP.java index 5e7a8360bf..3ff2b01800 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/FileDSP.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/FileDSP.java @@ -8,8 +8,10 @@ import dap4.core.util.DapException; import dap4.core.util.DapUtil; import dap4.dap4lib.serial.D4DSP; +import ucar.nc2.NetcdfFile; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.net.URISyntaxException; @@ -33,6 +35,8 @@ public class FileDSP extends D4DSP //Coverity[FB.URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD] protected byte[] raw = null; // Complete serialized binary databuffer + protected File file = null; + ////////////////////////////////////////////////// // Constructor(s) @@ -44,6 +48,8 @@ public FileDSP() ////////////////////////////////////////////////// // DSP API + public String getLocation() {return this.file.getAbsolutePath();} + /** * A path is file if it has no base protocol or is file: * @@ -75,21 +81,15 @@ public void close() @Override public FileDSP - open(String filepath) + open(File file) throws DapException { + this.file = file; try { - if(filepath.startsWith("file:")) try { - XURI xuri = new XURI(filepath); - filepath = xuri.getPath(); - } catch (URISyntaxException use) { - throw new DapException("Malformed filepath: " + filepath) - .setCode(DapCodes.SC_NOT_FOUND); - } - try (FileInputStream stream = new FileInputStream(filepath)) { + try (FileInputStream stream = new FileInputStream(file)) { this.raw = DapUtil.readbinaryfile(stream); } - try (FileInputStream stream = new FileInputStream(filepath)) { // == rewind + try (FileInputStream stream = new FileInputStream(file)) { // == rewind ChunkInputStream rdr = new ChunkInputStream(stream, RequestMode.DAP); String document = rdr.readDMR(); byte[] serialdata = DapUtil.readbinaryfile(rdr); diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/HttpDSP.java b/dap4/d4lib/src/main/java/dap4/dap4lib/HttpDSP.java index aa8becbe97..0a34b57efe 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/HttpDSP.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/HttpDSP.java @@ -47,7 +47,7 @@ public class HttpDSP extends D4DSP static protected final String QUERYSTART = "?"; static protected final String CONSTRAINTTAG = "dap4.ce"; - static protected final String PROTOTAG = "protocol"; + static protected final String[] MODETAGS = new String[]{"mode","proto","protocol"}; static protected final int DFALTPRELOADSIZE = 50000; // databuffer @@ -92,10 +92,16 @@ public HttpDSP() ////////////////////////////////////////////////// // DSP API + @Override + public String getLocation() + { + return xuri.toString(); + } + /** * A path is a DAP4 path if at least one of the following is true. * 1. it has "dap4:" as its leading protocol - * 2. it has #protocol=dap4 in its fragment + * 2. it has #mode=dap4 in its fragment * * @param url * @param context Any parameters that may help to decide. @@ -142,11 +148,10 @@ public boolean dspMatch(String url, DapContext context) } @Override - public HttpDSP open(String url) + public HttpDSP open(URI uri) throws DapException { - setLocation(url); - parseURL(url); + parseURL(uri.toString()); /* Take from the incoming data String s = xuri.getFragFields().get(Dap4Util.DAP4CSUMTAG); diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/netcdf/Nc4DSP.java b/dap4/d4lib/src/main/java/dap4/dap4lib/netcdf/Nc4DSP.java index d7209ac89b..1424283949 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/netcdf/Nc4DSP.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/netcdf/Nc4DSP.java @@ -25,6 +25,7 @@ import dap4.dap4lib.AbstractDSP; import dap4.dap4lib.DapCodes; import dap4.dap4lib.XURI; +import ucar.nc2.NetcdfFile; import ucar.nc2.jni.netcdf.Nc4Iosp; import ucar.nc2.jni.netcdf.Nc4prototypes; import ucar.nc2.jni.netcdf.SizeTByReference; @@ -301,10 +302,11 @@ protected void allnotesInit() protected int ncid = -1; // file id ; also set as DSP.source protected int format = 0; // from nc_inq_format protected int mode = 0; - protected String filepath = null; // real path to the dataset protected DMRFactory dmrfactory = null; + protected NetcdfFile ncfile = null; + ////////////////////////////////////////////////// // Constructor(s) @@ -324,24 +326,23 @@ public Nc4DSP() ////////////////////////////////////////////////// // DSP API + @Override + public String getLocation() + { + return ncfile.getLocation(); + } + @Override public Nc4DSP - open(String filepath) + open(NetcdfFile ncfile) throws DapException { - if(filepath.startsWith("file:")) try { - XURI xuri = new XURI(filepath); - filepath = xuri.getPath(); - } catch (URISyntaxException use) { - throw new DapException("Malformed filepath: " + filepath) - .setCode(DapCodes.SC_NOT_FOUND); - } int ret, mode; + this.ncfile = ncfile; IntByReference ncidp = new IntByReference(); - this.filepath = filepath; try { mode = NC_NOWRITE; - Nc4Cursor.errcheck(nc4, ret = nc4.nc_open(this.filepath, mode, ncidp)); + Nc4Cursor.errcheck(nc4, ret = nc4.nc_open(this.ncfile.getLocation(), mode, ncidp)); this.ncid = ncidp.getValue(); // Figure out what kind of file IntByReference formatp = new IntByReference(); @@ -349,7 +350,7 @@ public Nc4DSP() this.format = formatp.getValue(); if(DEBUG) System.out.printf("TestNetcdf: open: %s; ncid=%d; format=%d%n", - this.filepath, ncid, this.format); + this.ncfile.getLocation(), ncid, this.format); // Compile the DMR Nc4DMRCompiler dmrcompiler = new Nc4DMRCompiler(this, ncid, dmrfactory); setDMR(dmrcompiler.compile()); @@ -375,7 +376,7 @@ public void close() Nc4Cursor.errcheck(nc4, ret); closed = true; if(trace) - System.out.printf("Nc4DSP: closed: %s%n", this.filepath); + System.out.printf("Nc4DSP: closed: %s%n", this.ncfile.getLocation()); } @Override @@ -415,12 +416,6 @@ public Nc4prototypes getJNI() return this.nc4; } - @Override - public String getLocation() - { - return this.filepath; - } - ////////////////////////////////////////////////// // Utilities diff --git a/dap4/d4servlet/build.gradle b/dap4/d4servlet/build.gradle index 83914ca43b..72e035bacb 100644 --- a/dap4/d4servlet/build.gradle +++ b/dap4/d4servlet/build.gradle @@ -7,6 +7,7 @@ apply from: "$rootDir/gradle/any/publishing.gradle" dependencies { compile project(':dap4:d4core') compile project(':dap4:d4lib') + compile project(':dap4:d4cdm') compile project(":httpservices") compileOnly libraries["javax.servlet-api"] diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapCache.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapCache.java index b6b6746e76..374b1076f0 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapCache.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapCache.java @@ -3,16 +3,23 @@ package dap4.servlet; +import dap4.cdm.dsp.CDMDSP; import dap4.core.ce.CEConstraint; import dap4.core.data.DSP; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapContext; import dap4.core.util.DapException; -import dap4.dap4lib.DapCodes; +import dap4.dap4lib.FileDSP; +import dap4.dap4lib.HttpDSP; +import dap4.dap4lib.netcdf.Nc4DSP; +import ucar.nc2.NetcdfFile; +import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** @@ -34,43 +41,27 @@ abstract public class DapCache static final int MAXFILES = 100; // size of the cache - static public final String MATCHMETHOD = "dspMatch"; + ////////////////////////////////////////////////// + // Types ////////////////////////////////////////////////// // Static variables - /** - * Define a map of known DSP classes. - */ - static public DSPRegistry dspregistry = new DSPRegistry(); - /** * Define an lru cache of known DSP objects: oldest first. */ static protected List lru = new ArrayList(); - // This should be set by any subclass - static protected DSPFactory factory = null; - - static public void setFactory(DSPFactory f) - { - factory = f; - } - - static public DSPFactory getFactory() - { - return factory; - } - - static public synchronized DSP open(String path, DapContext cxt) + /**************************************************/ + /* Check cache */ + static protected DSP locateDSP(String location) throws IOException { - assert cxt != null; int lrusize = lru.size(); for(int i = lrusize - 1; i >= 0; i--) { DSP dsp = lru.get(i); String dsppath = dsp.getLocation(); - if(dsppath.equals(path)) { + if(dsppath.equals(location)) { // move to the front of the queue to maintain LRU property lru.remove(i); lru.add(dsp); @@ -78,22 +69,19 @@ static public synchronized DSP open(String path, DapContext cxt) return dsp; } } - // No match found, create and initialize it. + return null; /* no match found */ + } + + static void addDSP(DSP dsp) + throws DapException + { // If cache is full, remove oldest entry - if(lrusize == MAXFILES) { + if(lru.size() == MAXFILES) { // make room lru.remove(0); CEConstraint.release(lru.get(0).getDMR()); } - // Find dsp that can process this path - DSP dsp = dspregistry.findMatchingDSP(path,cxt); - if(dsp == null) - throw new DapException("Resource has no matching DSP: " + path) - .setCode(DapCodes.SC_FORBIDDEN); - dsp.setContext(cxt); - dsp.open(path); lru.add(dsp); - return dsp; } static synchronized public void flush() // for testing @@ -107,5 +95,20 @@ static synchronized public void flush() // for testing } } + /**************************************************/ + // DapDSP pass-thrus + + static public synchronized DSP open(DapRequest drq, NetcdfFile ncfile, DapContext cxt) + throws IOException + { + return DapDSP.open(drq,ncfile,cxt); + } + + static public synchronized DSP open(DapRequest drq, String target, DapContext cxt) + throws IOException + { + return DapDSP.open(drq,target,cxt); + } + } // DapCache diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapController.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapController.java index dc7d96ea0a..3933179cb0 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapController.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapController.java @@ -14,6 +14,7 @@ import dap4.core.dmr.ErrorResponse; import dap4.core.util.*; import dap4.dap4lib.*; +import ucar.nc2.NetcdfFile; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -116,16 +117,20 @@ public DapController() abstract protected void doCapabilities(DapRequest drq, DapContext cxt) throws IOException; /** - * Convert a URL path into an absolute file path - * Note that it is assumed than any leading servlet prefix has been removed. - * - * @param drq dap request - * @param location suffix of url path - * @return + * Get the root of the resource directory. + * In some cases, the root is relative to the Dap Request, so + * we include that as an argument. + * @param drq controlling dap request + * @return absolute path to resource directory root * @throws IOException */ - abstract public String getResourcePath(DapRequest drq, String location) throws DapException; + abstract public String getResourceRoot(DapRequest drq) throws DapException; + + /* + * Ask the controller if it can convert a string to a NetcdfFile + */ + abstract public NetcdfFile getNetcdfFile(DapRequest drq, String location) throws DapException; /** * Get the maximum # of bytes per request @@ -301,10 +306,11 @@ else if(t instanceof IOException) doDMR(DapRequest drq, DapContext cxt) throws IOException { - // Convert the url to an absolute path - String realpath = getResourcePath(drq, drq.getDatasetPath()); + String root = drq.getController().getResourceRoot(drq); + String location = drq.getDatasetPath(); + String path = DapUtil.canonjoin(root, location); - DSP dsp = DapCache.open(realpath, cxt); + DSP dsp = DapCache.open(drq, path, cxt); DapDataset dmr = dsp.getDMR(); /* Annotate with our endianness */ @@ -357,12 +363,13 @@ else if(t instanceof IOException) doData(DapRequest drq, DapContext cxt) throws IOException { - // Convert the url to an absolute path - String realpath = getResourcePath(drq, drq.getDatasetPath()); + String root = drq.getController().getResourceRoot(drq); + String location = drq.getDatasetPath(); + String path = DapUtil.canonjoin(root, location); - DSP dsp = DapCache.open(realpath, cxt); + DSP dsp = DapCache.open(drq, path, cxt); if(dsp == null) - throw new DapException("No such file: " + drq.getResourceRoot()); + throw new DapException("No such file: " + path); DapDataset dmr = dsp.getDMR(); if(DUMPDMR) { printDMR(dmr); diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java new file mode 100644 index 0000000000..1db3cf67e5 --- /dev/null +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java @@ -0,0 +1,315 @@ +/* Copyright 2012, UCAR/Unidata. + See the LICENSE file for more information. */ + +package dap4.servlet; + +import dap4.cdm.dsp.CDMDSP; +import dap4.core.ce.CEConstraint; +import dap4.core.data.DSP; +import dap4.core.util.DapContext; +import dap4.core.util.DapException; +import dap4.dap4lib.AbstractDSP; +import dap4.dap4lib.DapCodes; +import dap4.dap4lib.FileDSP; +import dap4.dap4lib.HttpDSP; +import dap4.dap4lib.netcdf.Nc4DSP; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + + +/** +The old DAP4 code assumed that all source files +on the server could be specified using a path string. +It turns out that for virtual files like .ncml files, +this is false and instead, a NetcdfFile instance must be used. + +Fixing this necessitated some significant changes to the DAP4 code. +The basic fix was to make the primary DAP4 code operate with respect +to a DSP object. Then the problem was to change the code that invoked DAP4 +so that it figured out what kind of DSP object to use. + +The primary change on the server side is to provide a function that, +given an object, can figure out what kind of DSP to use to convert that +kind of object to a DAP4 representation. This is defined in the new class +called DapDSP. DapDSP has two primary externally visible API functions: +1. open a NetcdfFile - returns a CDMDSP wrapping the NetcdfFile object. +2. open a String - uses the rules below to figure out what kind of DSP + to return. + +If the string parses as a URI, then look at the protocol. +If the protocol is file://, then treat as a file path and +use the rules below. + +Otherwise, look at the protocol and the mode arg in the fragment +part and use the following mapping. + * dap4://... use HttpDSP with URI converted to http://... + * #mode=dap4 - use HttpDSP + * #dap4 - use HttpDSP +In practice, HttpDSP is only used on the client side to read a +DAP4 encoded stream. + +If the String is a file path (or file:// URL) then +determine the DSP using the file path extension as follows: + * .dmr|.syn| - use SynDSP + * .dap|.raw - use FileDSP + * .nc|.hdf5 - use Nc4DSP + +As a rule, clients will end up using HttpDSP to read the +incoming DAP4 encode Http stream. Again as a rule, Servers will +assume an incoming Http request is for a NetcdfFile and will use +CDMDSP to convert that to a DAP4 Http stream. In the event that +Server detects that the file to be read is .nc, then it may +choose to use Nc4DSP to read the file, although it is not +required to and can instead open the file as a NetcdfFile object +and use CDMDSP. All of the other cases are, again as a rule, +used for testing and can be identified by the special path/URI extension +(e.g. .syn or .dap or .raw). + +The other major change, and the one that actually prompted this +whole change, is in the DapController and its +subclasses. Specifically, the function getNetcdfFile() has been +added to attempt to convert a String (a location) to a +NetcdfFile object (ultimately via DapDSP). It is only if this +fails that an attempt is made to look for the other cases via +the location extension. + */ + +abstract public class DapDSP +{ + + ////////////////////////////////////////////////// + // Constants + + static final String driveletters = "abcdefghijklmnopqrstuvwxyz" +"abcdefghijklmnopqrstuvwxyz".toUpperCase(); + + ////////////////////////////////////////////////// + // Types + + static protected class ExtMatch // match by extension + { + String ext; + Class dsp; + + public ExtMatch(String ext, Class dsp) { + this.ext = ext; + this.dsp = dsp; + } + } + + static protected class FragMatch // match by URL fragment keys + { + String key; + String value; + Class dsp; + + public FragMatch(String key, String value, Class dsp) + { + this.key = key; + this.value = value; + this.dsp = dsp; + } + } + + static protected class ProtocolMatch // match by URL protocol + { + String proto; + String replace; + Class dsp; + + public ProtocolMatch(String proto, String replace, Class dsp) + { + this.proto = proto; + this.replace = replace; + this.dsp = dsp; + } + } + + ////////////////////////////////////////////////// + // Static variables + + static ExtMatch[] match = new ExtMatch[]{ + new ExtMatch(".dmr", SynDSP.class), + new ExtMatch(".syn", SynDSP.class), + new ExtMatch(".nc", Nc4DSP.class), + new ExtMatch(".hdf5", Nc4DSP.class), + new ExtMatch(null, FileDSP.class) // default + }; + + static FragMatch[] frags = new FragMatch[]{ + new FragMatch("mode", "dap4", HttpDSP.class), + new FragMatch("proto", "dap4", HttpDSP.class), + new FragMatch("protocol", "dap4", HttpDSP.class), + new FragMatch("dap4", null, HttpDSP.class), + }; + + static ProtocolMatch[] protos = new ProtocolMatch[]{ + new ProtocolMatch("dap4", "https", HttpDSP.class), + }; + + /**************************************************/ + // Provide versions of open corresponding to the source type + static protected DSP open(NetcdfFile ncfile, DapContext cxt) + throws IOException + { + // Convert to CDMDSP + CDMDSP dsp = new CDMDSP(); + if(dsp != null) dsp.setContext(cxt); + dsp.open(ncfile); + return dsp; + } + + // Open a File + static protected DSP open(File file, DapContext cxt) + throws IOException + { + // Choose the DSP based on the file extension + String path = file.getPath(); + Class cldsp = null; + for(ExtMatch em : match) { + if(em.ext == null) { + cldsp = em.dsp; + break; + } + if(path.endsWith(em.ext)) {cldsp = em.dsp; break;} + } + if(cldsp == null) + throw new DapException("Indeciperable file: " + file); + AbstractDSP dsp = null; + try { + dsp = (AbstractDSP) cldsp.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IOException("Class instance creation failed", e); + } + if(dsp != null) dsp.setContext(cxt); + dsp.open(file); + return dsp; + } + + // Open a URL + static protected DSP open(URI uri, DapContext cxt) + throws IOException + { + AbstractDSP dsp = null; + // See if this is a DAP4 url + Class cldsp = null; + // Search the fragment list for markers + String fragments = uri.getFragment(); + Map fragmap = parsefragment(fragments); + for(FragMatch fm : frags) { + String values = fragmap.get(fm.key); + if(values != null) { + if(fm.value == null) { + cldsp = fm.dsp; + break; + } // singleton case + else {// search for match + if(values.indexOf(fm.key) >= 0) { + cldsp = fm.dsp; + break; + } + } + } + if(cldsp == null) + throw new DapException("Indeciperable URI: " + uri); + try { + dsp = (AbstractDSP) cldsp.newInstance(); + break; + } catch (InstantiationException | IllegalAccessException e) { + throw new IOException("Class instance creation failed", e); + } + } + if(dsp != null) { + dsp.setContext(cxt); + dsp.open(uri); + } + return dsp; + } + + /**************************************************/ + // Exported versions for NetcdfFile and String + + static public synchronized DSP open(DapRequest drq, NetcdfFile ncfile, DapContext cxt) + throws IOException + { + assert cxt != null && ncfile != null; + // Convert to CDMDSP + DSP dsp = open(ncfile,cxt); + return dsp; + } + + static public synchronized DSP open(DapRequest drq, String target, DapContext cxt) + throws IOException + { + assert cxt != null && target != null; + String path = null; + DSP dsp = null; + + // See if this parses as a URL + try { + URI uri = new URI(target); + // Windows drive letters cause URI to succeed, so special hack for that + if (uri.getScheme().length() == 1 && driveletters.indexOf(uri.getScheme().charAt(0)) >= 0) + throw new URISyntaxException("windows drive letter",target); + // If uri protocol is file, then extract the path + if(uri.getScheme().equals("file")) + path = uri.getPath(); + else + dsp = open(uri,cxt); // open as general URI + } catch (URISyntaxException use) { + // assume it is a simple file path + path = target; + } + + if(dsp == null) { + // See if this can open as a NetcdfFile|NetcdfDataset + NetcdfFile ncfile = null; + try { + ncfile = drq.getController().getNetcdfFile(drq,path); + } catch (IOException ioe) { + ncfile = null; + } + if(ncfile != null) { + dsp = open(ncfile,cxt); + } + } + + if(dsp == null) { + // Finally, try to open as a some kind of File object + File file = new File(path); + // Complain if it does not exist + if(!file.exists()) + throw new DapException("Not found: " + target) + .setCode(DapCodes.SC_NOT_FOUND); + else + dsp = open(file,cxt); + if(dsp != null) dsp.setContext(cxt); + ((AbstractDSP)dsp).open(file); + } + return dsp; + } + + static protected Map + parsefragment(String fragments) + { + String[] pieces = fragments.split("[&]"); + Map map = new HashMap<>(); + for(String p : pieces) { + String[] pair = p.split("[=]"); + if(pair.length == 1) { + map.put(pair[0].trim(), ""); + } else { + map.put(pair[0].trim(), pair[1].trim()); + } + } + + return map; + } + + +} // DapCache diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java index 7e08e3c1f5..6cc4740087 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java @@ -321,6 +321,11 @@ public String getControllerPath() return this.controllerpath; } + public DapController getController() + { + return this.controller; + } + public String getURLPath() { return this.controllerpath + (this.datasetpath == null ? "" : this.datasetpath); @@ -358,12 +363,6 @@ public String queryLookup(String name) return this.queries; } - public String getResourcePath(String relpath) - throws IOException - { - return controller.getResourcePath(this, relpath); - } - public String getDatasetPath() { return this.datasetpath; diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/SynDSP.java b/dap4/d4servlet/src/main/java/dap4/servlet/SynDSP.java index bc9ffbf388..71debad7b1 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/SynDSP.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/SynDSP.java @@ -15,10 +15,7 @@ import dap4.dap4lib.XURI; import dap4.dap4lib.serial.D4DSP; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.IOException; +import java.io.*; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -40,6 +37,8 @@ public class SynDSP extends D4DSP protected byte[] raw = null; // Complete serialized binary databuffer + protected File file = null; + ////////////////////////////////////////////////// // Constructor(s) @@ -51,6 +50,8 @@ public SynDSP() ////////////////////////////////////////////////// // DSP API + @Override + public String getLocation() {return this.file.getAbsolutePath();} /** * A path is a Synthetic path if it ends in .dmr or .syn @@ -75,23 +76,16 @@ public void close() @Override public SynDSP - open(String filepath) + open(File file) throws DapException { // convert the relative path to real path assert this.context != null; - if(filepath.startsWith("file:")) try { - XURI xuri = new XURI(filepath); - filepath = xuri.getPath(); - } catch (URISyntaxException use) { - throw new DapException("Malformed filepath: " + filepath) - .setCode(DapCodes.SC_NOT_FOUND); - } - setLocation(filepath); + this.file = file; // Read the .dmr/.syn file String document; try { - try (FileInputStream stream = new FileInputStream(filepath);) { + try (FileInputStream stream = new FileInputStream(file);) { document = DapUtil.readtextfile(stream); } } catch (IOException ioe) { diff --git a/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dmr b/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dmr new file mode 100644 index 0000000000..cbf9745570 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dmr @@ -0,0 +1,16 @@ + + + + + + + + + + + diff --git a/dap4/d4tests/src/test/data/resources/testfiles/test_ncml.ncml b/dap4/d4tests/src/test/data/resources/testfiles/test_ncml.ncml new file mode 100644 index 0000000000..c0913c682f --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/testfiles/test_ncml.ncml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/dap4/d4tests/src/test/java/dap4/test/D4TSController.java b/dap4/d4tests/src/test/java/dap4/test/D4TSController.java index 0a500e01c2..c2159b111b 100644 --- a/dap4/d4tests/src/test/java/dap4/test/D4TSController.java +++ b/dap4/d4tests/src/test/java/dap4/test/D4TSController.java @@ -12,6 +12,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; +import ucar.nc2.NetcdfFile; +import ucar.nc2.dataset.NetcdfDataset; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -49,12 +51,12 @@ public void handleRequest(HttpServletRequest req, HttpServletResponse res) super.handleRequest(req, res); } - @Override + //Not used public String getResourcePath(DapRequest drq, String location) throws DapException { - String prefix = drq.getResourceRoot(); + String prefix = getResourceRoot(drq); if(prefix == null) throw new DapException("Cannot find location resource: " + location) .setCode(DapCodes.SC_NOT_FOUND); @@ -74,4 +76,41 @@ public void handleRequest(HttpServletRequest req, HttpServletResponse res) return datasetfilepath; } + @Override + public String + getResourceRoot(DapRequest drq) + throws DapException + { + String rootpath = drq.getResourceRoot(); + if(rootpath == null) + throw new DapException("Cannot find resource root") + .setCode(DapCodes.SC_NOT_FOUND); + // See if it really exists and is readable and of proper type + File root = new File(rootpath); + if(!root.exists() || !root.canRead() || !root.isDirectory()) + throw new DapException("Resource root path does not exist") + .setCode(HttpServletResponse.SC_NOT_FOUND); + return rootpath; + } + + /* + * Ask the controller if it can convert a string to a NetcdfFile + */ + @Override + public NetcdfFile getNetcdfFile(DapRequest drq, String path) + { + NetcdfFile ncfile = null; + try { + ncfile = NetcdfFile.open(path); + } catch (IOException io1) { + try { + // Try as netcdfdataset + ncfile = NetcdfDataset.openFile(path, null); + } catch (IOException io2) { + ncfile = null; + } + } + return ncfile; + } + } diff --git a/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java b/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java index 5ed317c13e..e6e6bda691 100644 --- a/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java +++ b/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java @@ -4,7 +4,6 @@ package dap4.test; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapException; import dap4.core.util.DapUtil; import dap4.dap4lib.FileDSP; @@ -489,8 +488,6 @@ public String getResourceDir() testSetup() { DapController.TESTING = true; - DapCache.dspregistry.register(FileDSP.class, DSPRegistry.FIRST); - DapCache.dspregistry.register(SynDSP.class, DSPRegistry.FIRST); try { // Always prefer Nc4Iosp over HDF5 NetcdfFile.iospDeRegister(NC4IOSP); diff --git a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java index 3339c8e173..50e8ca8a68 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java @@ -39,7 +39,7 @@ public class TestCDMClient extends DapTestCommon static final String BASEEXTENSION = ".txt"; static final String INPUTEXTENSION = ".raw"; - static final String DAP4TAG = "#protocol=dap4"; + static final String DAP4TAG = "#mode=dap4"; static final String DATADIR = "src/test/data/resources"; // relative to dap4 root static final String BASELINEDIR = "TestCDMClient/baseline"; @@ -187,7 +187,7 @@ public void setup() throws Exception chooseTestcases() { if(false) { - chosentests = locate("file:", "test_atomic_array.nc.raw"); + chosentests = locate("file:", "test_ncml.ncml.raw"); prop_visual = true; prop_baseline = false; } else { diff --git a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java index 7efb6ff982..e1b9d70f12 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java @@ -23,6 +23,8 @@ import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -47,7 +49,7 @@ public class TestDSP extends DapTestCommon static final String BASEEXTENSION = "txt"; - static final String DAP4TAG = "#protocol=dap4"; + static final String DAP4TAG = "#mode=dap4"; ////////////////////////////////////////////////// // Constants @@ -147,31 +149,32 @@ public String toString() protected DSP dspFor(String surl) { - URL url; + URI uri; try { - url = new URL(surl); - } catch (MalformedURLException mue) { + uri = new URI(surl); + } catch (URISyntaxException mue) { throw new IllegalArgumentException("Malformed url: " + surl); } - String proto = url.getProtocol(); - String path = url.getPath(); + String proto = uri.getScheme(); + String path = uri.getPath(); int dot = path.lastIndexOf('.'); if(dot < 0) dot = path.length(); String ext = path.substring(dot, path.length()); DSP dsp = null; try { if("file".equals(proto)) { + File f = new File(path); // discriminate on the extensions if(".raw".equals(ext)) { - dsp = new FileDSP(); + dsp = new FileDSP().open(f); } else if(".syn".equals(ext)) { - dsp = new SynDSP(); + dsp = new SynDSP().open(f); } if(".nc".equals(ext)) { - dsp = new Nc4DSP(); + dsp = new Nc4DSP().open(f); } } else if("http".equals(proto) - || "https".equals(url.getProtocol())) { - dsp = new HttpDSP(); + || "https".equals(proto)) { + dsp = new HttpDSP().open(uri); } else throw new IllegalArgumentException("Cannot determine DSP class for: " + surl); } catch (DapException de) { @@ -321,9 +324,7 @@ public void testDSP() System.err.println("Baseline: " + testcase.getBaseline()); DSP dsp = dspFor(testcase.getURL()); - dsp.setContext(new DapContext()); - dsp.open(testcase.getURL()); String metadata = dumpmetadata(dsp); if(prop_visual) diff --git a/dap4/d4tests/src/test/java/dap4/test/TestDSR.java b/dap4/d4tests/src/test/java/dap4/test/TestDSR.java index fd26d33fd4..97b9da0462 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestDSR.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestDSR.java @@ -1,7 +1,6 @@ package dap4.test; -import dap4.core.data.DSPRegistry; import dap4.dap4lib.FileDSP; import dap4.servlet.DapCache; import dap4.servlet.Generator; @@ -62,8 +61,6 @@ public void setup() mvcbuilder.setValidator(new TestServlet.NullValidator()); this.mockMvc = mvcbuilder.build(); testSetup(); - DapCache.dspregistry.register(FileDSP.class, DSPRegistry.FIRST); - DapCache.dspregistry.register(SynDSP.class, DSPRegistry.FIRST); if(prop_ascii) Generator.setASCII(true); this.resourceroot = getResourceRoot(); diff --git a/dap4/d4tests/src/test/java/dap4/test/TestFrontPage.java b/dap4/d4tests/src/test/java/dap4/test/TestFrontPage.java index a00d604381..ddcedccc3a 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestFrontPage.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestFrontPage.java @@ -1,6 +1,5 @@ package dap4.test; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapUtil; import dap4.dap4lib.FileDSP; import dap4.servlet.DapCache; @@ -63,8 +62,6 @@ public void setup() throws Exception mvcbuilder.setValidator(new TestServlet.NullValidator()); this.mockMvc = mvcbuilder.build(); testSetup(); - DapCache.dspregistry.register(FileDSP.class, DSPRegistry.FIRST); - DapCache.dspregistry.register(SynDSP.class, DSPRegistry.FIRST); if(prop_ascii) Generator.setASCII(true); this.resourceroot = getResourceRoot(); diff --git a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java index 3765ae00b0..58d609da88 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java @@ -328,21 +328,5 @@ String ncdumpdata(NetcdfDataset ncfile, String datasetname) System.err.println(msg); return false; } -//////////////////////////////////////// - // Stand alone - - static public void - main(String[] argv) - { - try { - new TestConstraints().testConstraints(); - } catch (Exception e) { - System.err.println("*** FAIL"); - e.printStackTrace(); - System.exit(1); - } - System.err.println("*** PASS"); - System.exit(0); - }// main } // class TestConstraints diff --git a/dap4/d4tests/src/test/java/dap4/test/TestServlet.java b/dap4/d4tests/src/test/java/dap4/test/TestServlet.java index 8acc16a626..11c5bbf05a 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestServlet.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestServlet.java @@ -205,7 +205,7 @@ public void cleanup() chooseTestcases() { if(false) { - chosentests = locate("test_struct_type.nc"); + chosentests = locate("test_sequence_1.syn"); prop_visual = true; prop_generate = false; prop_baseline = false; @@ -281,7 +281,12 @@ public void testServlet() byte[] byteresult = res.getContentAsByteArray(); // Test by converting the raw output to a string - String sdmr = new String(byteresult, UTF8); + String charencode = res.getCharacterEncoding(); + String sdmr = null; + if(charencode == "ISO-8859-1") + sdmr = new String(byteresult, ISO88591); + else + sdmr = new String(byteresult, UTF8); if(prop_visual) visual(testcase.title + ".dmr", sdmr); @@ -912,6 +917,27 @@ public void run(Dump printer) throws IOException printer.verifychecksum(); } })); + this.alltestcases.add( + new TestCase("test_ncml.ncml", "dmr,dap", true, //8 + // S4 U1 S2 S4 + new Dump.Commands() + { + public void run(Dump printer) throws IOException + { + printer.startchecksum(); + printer.printvalue('S', 4); + printer.verifychecksum(); + printer.startchecksum(); + printer.printvalue('U', 1); + printer.verifychecksum(); + printer.startchecksum(); + printer.printvalue('S', 2); + printer.verifychecksum(); + printer.startchecksum(); + printer.printvalue('S', 4); + printer.verifychecksum(); + } + })); // XFAIL tests this.alltestcases.add( new TestCase("test_struct_array.nc", "dmr", true) diff --git a/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java b/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java index 143e3909a1..cd5c1b08bb 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java @@ -1,7 +1,6 @@ package dap4.test; import dap4.core.ce.parser.CEParserImpl; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapDump; import dap4.core.util.Escape; import dap4.dap4lib.ChunkInputStream; diff --git a/dap4/d4ts/src/main/java/dap4/d4ts/D4TSServlet.java b/dap4/d4ts/src/main/java/dap4/d4ts/D4TSServlet.java index 804ac5fcd1..4c28f7ff71 100644 --- a/dap4/d4ts/src/main/java/dap4/d4ts/D4TSServlet.java +++ b/dap4/d4ts/src/main/java/dap4/d4ts/D4TSServlet.java @@ -4,7 +4,6 @@ package dap4.d4ts; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapContext; import dap4.core.util.DapException; import dap4.core.util.DapUtil; @@ -13,6 +12,7 @@ import dap4.dap4lib.FileDSP; import dap4.dap4lib.netcdf.Nc4DSP; import dap4.servlet.*; +import ucar.nc2.NetcdfFile; import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -43,28 +43,6 @@ public class D4TSServlet extends DapController ////////////////////////////////////////////////// // Type Decls - static class D4TSFactory extends DSPFactory - { - - public D4TSFactory() - { - // Register known DSP classes: order is important - // in event that two or more dsps can match a given file - // (e.q. FileDSP vs Nc4DSP). - // Only used in server - DapCache.dspregistry.register(Nc4DSP.class, DSPRegistry.LAST); - DapCache.dspregistry.register(SynDSP.class, DSPRegistry.LAST); - DapCache.dspregistry.register(FileDSP.class, DSPRegistry.LAST); - } - - } - - ////////////////////////////////////////////////// - - static { - DapCache.setFactory(new D4TSFactory()); - } - ////////////////////////////////////////////////// // Instance variables @@ -105,7 +83,8 @@ protected void doGet(HttpServletRequest req, throws IOException { DapRequest drq = (DapRequest)cxt.get(DapRequest.class); - String favfile = getResourcePath(drq, icopath); + String prefix = getResourceRoot(drq); + String favfile = DapUtil.canonjoin(prefix, icopath); if(favfile != null) { try (FileInputStream fav = new FileInputStream(favfile);) { byte[] content = DapUtil.readbinaryfile(fav); @@ -138,14 +117,12 @@ protected void doGet(HttpServletRequest req, } @Override - public String - getResourcePath(DapRequest drq, String location) + public NetcdfFile + getNetcdfFile(DapRequest drq, String location) throws DapException { - String prefix = drq.getResourceRoot(); - if(prefix == null) - throw new DapException("Cannot find location resource: " + location) - .setCode(DapCodes.SC_NOT_FOUND); + NetcdfFile ncfile = null; + String prefix = getResourceRoot(drq); location = DapUtil.canonicalpath(location); String datasetfilepath = DapUtil.canonjoin(prefix, location); // See if it really exists and is readable and of proper type @@ -159,7 +136,26 @@ protected void doGet(HttpServletRequest req, if(!dataset.canRead()) throw new DapException("Requested file not readable: " + datasetfilepath) .setCode(HttpServletResponse.SC_FORBIDDEN); - return datasetfilepath; + try { + ncfile = NetcdfFile.open(datasetfilepath); + } catch (IOException ioe) { + ncfile = null; + } + return ncfile; + } + + @Override + public String + getResourceRoot(DapRequest drq) + throws DapException + { + String rootpath; + rootpath = drq.getResourceRoot(); + File f = (rootpath == null ? null : new File(rootpath)); + if(f == null || !f.exists() || !f.canRead() || !f.isDirectory()) + throw new DapException("Resource root path not found") + .setCode(DapCodes.SC_NOT_FOUND); + return rootpath; } @Override @@ -190,7 +186,7 @@ public long getBinaryWriteLimit() // Figure out the directory containing // the files to display. String pageroot; - pageroot = getResourcePath(drq, ""); + pageroot = getResourceRoot(drq); if(pageroot == null) throw new DapException("Cannot locate resources directory"); this.defaultroots = new ArrayList<>(); @@ -200,4 +196,4 @@ public long getBinaryWriteLimit() return new FrontPage(this.defaultroots, drq); } -} \ No newline at end of file +} diff --git a/tds/src/main/java/thredds/server/dap4/Dap4Controller.java b/tds/src/main/java/thredds/server/dap4/Dap4Controller.java index 934dcf7981..7ce6da7dd2 100644 --- a/tds/src/main/java/thredds/server/dap4/Dap4Controller.java +++ b/tds/src/main/java/thredds/server/dap4/Dap4Controller.java @@ -5,14 +5,10 @@ package thredds.server.dap4; -import dap4.core.data.DSPRegistry; import dap4.core.util.DapContext; import dap4.core.util.DapException; import dap4.core.util.DapUtil; import dap4.dap4lib.DapCodes; -import dap4.dap4lib.DapLog; -import dap4.servlet.DSPFactory; -import dap4.servlet.DapCache; import dap4.servlet.DapController; import dap4.servlet.DapRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -20,6 +16,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import thredds.core.TdsRequestedDataset; import ucar.nc2.NetcdfFile; +import ucar.nc2.dataset.NetcdfDataset; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -44,23 +41,6 @@ public class Dap4Controller extends DapController ////////////////////////////////////////////////// // Type Decls - static class Dap4Factory extends DSPFactory - { - - public Dap4Factory() - { - // For TDS, we only need to register one DSP type: ThreddsDSP. - // This is because we will always serve only NetcdfFile objects. - // See D4TSServlet for a multiple registration case. - DapCache.dspregistry.register(ThreddsDSP.class, DSPRegistry.LAST); - } - - } - - static { - DapCache.setFactory(new Dap4Factory()); - } - ////////////////////////////////////////////////// // Spring Elements @@ -118,44 +98,71 @@ public Dap4Controller() return "dap4"; } - // There is a problem Spring under intellij when using mocking. - // See TestServlet for more info. In any case, if autowiring does - // not work, then TdsRequestedDataset.getLocationFromRequestPath - // will fail because it internal DatasetManager value will be null. - // Autowiring would have set it to non-null. So, check to see if - // the autowiring worked and if so use - // TdsRequestedDataset.getLocationFromRequestPath. - // Otherwise, compute the proper path from the drq.getResourceRoot. - // This is completely a hack until such time as we can get things - // to work under Intellij. @Override public String - getResourcePath(DapRequest drq, String location) + getResourceRoot(DapRequest drq) throws DapException { - String realpath; + String rootpath; if(TdsRequestedDataset.getDatasetManager() != null) { - realpath = TdsRequestedDataset.getLocationFromRequestPath(location); + rootpath = TdsRequestedDataset.getLocationFromRequestPath(""); } else { assert TdsRequestedDataset.getDatasetManager() == null; - String prefix = drq.getResourceRoot(); - assert (prefix != null); - realpath = DapUtil.canonjoin(prefix, location); + rootpath = drq.getResourceRoot(); } + // Root path must exist + File f = (rootpath == null ? null : new File(rootpath)); + if(f == null || !f.exists() || !f.canRead() || !f.isDirectory()) + throw new DapException("Resource root path not found") + .setCode(DapCodes.SC_NOT_FOUND); + return rootpath; + } - if (!TESTING) { - if (!TdsRequestedDataset.resourceControlOk(drq.getRequest(), drq.getResponse(), realpath)) - throw new DapException("Not authorized: " + location) + /* + * There is a problem with Spring under intellij when using mocking. + * See TestServlet for more info. In any case, if autowiring does + * not work, then TdsRequestedDataset.getLocationFromRequestPath + * will fail because its internal DatasetManager value will be null. + * Autowiring would have set it to non-null. So, check to see if + * the autowiring worked and if so use one of two different mechanisms. + * This is completely a hack until such time as we can get things + * to work under Intellij. + */ +/* + *Ask the controller if it can convert a string to a NetcdfFile + */ + @Override + public NetcdfFile getNetcdfFile(DapRequest drq, String path) + throws DapException + { + NetcdfFile ncfile = null; + if(TdsRequestedDataset.getDatasetManager() != null) { + try { + ncfile = TdsRequestedDataset.getNetcdfFile(drq.getRequest(), drq.getResponse(), path); + } catch (IOException ioe) { + ncfile = null; + } + } else { + assert TdsRequestedDataset.getDatasetManager() == null; + try { + ncfile = NetcdfFile.open(path); + } catch (IOException io1) { + try { + // Try as netcdfdataset + ncfile = NetcdfDataset.openFile(path, null); + } catch (IOException io2) { + ncfile = null; + } + } + } + if(ncfile != null && !TESTING) { + if(!TdsRequestedDataset.resourceControlOk(drq.getRequest(), drq.getResponse(), + ncfile.getLocation())) + throw new DapException("Not authorized: " + path) .setCode(DapCodes.SC_FORBIDDEN); } - File f = new File(realpath); - if (!f.exists() || !f.canRead()) - throw new DapException("Not found: " + location) - .setCode(DapCodes.SC_NOT_FOUND); - //ncfile = TdsRequestedDataset.getNetcdfFile(this.request, this.response, path); - return realpath; + return ncfile; } } - diff --git a/tds/src/main/java/thredds/server/dap4/ThreddsDSP.java b/tds/src/main/java/thredds/server/dap4/ThreddsDSP.java deleted file mode 100644 index 5c00a3a192..0000000000 --- a/tds/src/main/java/thredds/server/dap4/ThreddsDSP.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 1998-2015 the University Corporation for Atmospheric Research/Unidata - * - * Portions of this software were developed by the Unidata Program at the - * University Corporation for Atmospheric Research. - * - * Access and use of this software shall impose the following obligations - * and understandings on the user. The user is granted the right, without - * any fee or cost, to use, copy, modify, alter, enhance and distribute - * this software, and any derivative works thereof, and its supporting - * documentation for any purpose whatsoever, provided that this entire - * notice appears in all copies of the software, derivative works and - * supporting documentation. Further, UCAR requests that the user credit - * UCAR/Unidata in any publications that result from the use of this - * software or in any product that includes this software. The names UCAR - * and/or Unidata, however, may not be used in any advertising or publicity - * to endorse or promote any products or commercial entity unless specific - * written permission is obtained from UCAR/Unidata. The user also - * understands that UCAR/Unidata is not obligated to provide the user with - * any support, consulting, training or assistance of any kind with regard - * to the use, operation and performance of this software nor to provide - * the user with any updates, revisions, new versions or "bug fixes." - * - * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING - * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package thredds.server.dap4; - -import dap4.cdm.dsp.CDMDSP; -import dap4.core.util.DapContext; -import dap4.core.util.DapException; -import dap4.core.util.DapUtil; -import dap4.dap4lib.DapCodes; -import thredds.core.TdsRequestedDataset; -import ucar.nc2.NetcdfFile; - -import java.io.File; - -/** - * CDM->DAP DSP - * Used (for now) only on server side. - */ - -public class ThreddsDSP extends CDMDSP -{ - - - ////////////////////////////////////////////////// - // Instance variables - - ////////////////////////////////////////////////// - // Constructor(s) - - public ThreddsDSP() - { - } - - public ThreddsDSP(String path) - throws DapException - { - super(path); - } - - ////////////////////////////////////////////////// - -} diff --git a/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java b/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java index 10a904dd52..f24ff5ca94 100644 --- a/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java +++ b/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java @@ -32,6 +32,7 @@ abstract public class UnitTestCommon static public final boolean DEBUG = false; static public final Charset UTF8 = Charset.forName("UTF-8"); + static public final Charset ISO88591 = Charset.forName("ISO-8859-1"); static protected final int[] OKCODES = new int[]{200, 404}; From d47eb0c74810407fbde083990f927ec992b36b9c Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 19 Oct 2019 16:39:11 -0600 Subject: [PATCH 3/7] Fix test cases to match other changes --- .../main/java/dap4/cdm/nc2/DapNetcdfFile.java | 3 +- .../src/main/java/dap4/core/util/Escape.java | 56 ++- .../src/main/java/dap4/dap4lib/XURI.java | 92 +++-- .../src/main/java/dap4/servlet/DapDSP.java | 41 ++- .../main/java/dap4/servlet/DapRequest.java | 4 +- .../TestDSP/baseline/test_ncml.ncml.raw.txt | 20 + .../baseline/test_sequence_1.syn.1.dap | 60 ++- .../baseline/test_sequence_1.syn.2.dap | 60 ++- .../baseline/test_atomic_array.nc.hdf5.dap | 57 +++ .../baseline/test_atomic_array.nc.hdf5.dmr | 31 ++ .../baseline/test_atomic_array.nc.nc4.dap | 107 +++--- .../baseline/test_atomic_array.nc.nc4.dmr | 60 +-- .../baseline/test_atomic_types.nc.hdf5.dap | 58 +++ .../baseline/test_atomic_types.nc.hdf5.dmr | 40 ++ .../baseline/test_atomic_types.nc.nc4.dap | 113 +++--- .../baseline/test_atomic_types.nc.nc4.dmr | 78 ++-- .../TestIosp/baseline/test_enum.nc.hdf5.dap | 14 + .../TestIosp/baseline/test_enum.nc.hdf5.dmr | 11 + .../TestIosp/baseline/test_enum.nc.nc4.dap | 25 +- .../TestIosp/baseline/test_enum.nc.nc4.dmr | 20 +- .../baseline/test_enum_array.nc.hdf5.dap | 17 + .../baseline/test_enum_array.nc.hdf5.dmr | 13 + .../baseline/test_enum_array.nc.nc4.dap | 31 +- .../baseline/test_enum_array.nc.nc4.dmr | 24 +- .../baseline/test_one_var.nc.hdf5.dap | 10 + .../baseline/test_one_var.nc.hdf5.dmr | 7 + .../TestIosp/baseline/test_one_var.nc.nc4.dap | 5 +- .../baseline/test_one_vararray.nc.hdf5.dap | 13 + .../baseline/test_one_vararray.nc.hdf5.dmr | 9 + .../baseline/test_one_vararray.nc.nc4.dap | 7 +- .../baseline/test_struct_array.nc.hdf5.dap | 68 ++++ .../baseline/test_struct_array.nc.hdf5.dmr | 17 + .../baseline/test_struct_array.nc.nc4.dap | 101 ++--- .../baseline/test_struct_nested.nc.hdf5.dap | 39 ++ .../baseline/test_struct_nested.nc.hdf5.dmr | 24 ++ .../baseline/test_struct_nested.nc.nc4.dap | 23 +- .../baseline/test_struct_type.nc.hdf5.dap | 21 ++ .../baseline/test_struct_type.nc.hdf5.dmr | 14 + .../baseline/test_struct_type.nc.nc4.dap | 12 +- .../TestIosp/baseline/test_vlen1.nc.hdf5.dap | 12 + .../TestIosp/baseline/test_vlen1.nc.hdf5.dmr | 8 + .../TestIosp/baseline/test_vlen1.nc.nc4.dap | 7 +- .../TestIosp/baseline/test_vlen2.nc.hdf5.dap | 22 ++ .../TestIosp/baseline/test_vlen2.nc.hdf5.dmr | 11 + .../TestIosp/baseline/test_vlen2.nc.nc4.dap | 21 +- .../TestIosp/baseline/test_vlen3.nc.hdf5.dap | 18 + .../TestIosp/baseline/test_vlen3.nc.hdf5.dmr | 11 + .../TestIosp/baseline/test_vlen3.nc.nc4.dap | 11 +- .../TestIosp/baseline/test_vlen4.nc.hdf5.dap | 21 ++ .../TestIosp/baseline/test_vlen4.nc.hdf5.dmr | 11 + .../TestIosp/baseline/test_vlen4.nc.nc4.dap | 17 +- .../TestIosp/baseline/test_vlen5.nc.hdf5.dap | 24 ++ .../TestIosp/baseline/test_vlen5.nc.hdf5.dmr | 13 + .../TestIosp/baseline/test_vlen5.nc.nc4.dap | 21 +- .../TestParsers/baseline/test_ncml.ncml.dmp | 15 + .../testfiles/dmr/test_atomic_types.nc.dmr | 8 +- .../testfiles/dmr/test_numeric_types.nc.dmr | 4 +- .../test/java/dap4/test/DapTestCommon.java | 16 +- .../src/test/java/dap4/test/GenerateRaw.java | 2 +- .../test/java/dap4/test/TestCDMClient.java | 4 +- .../test/java/dap4/test/TestConstraints.java | 9 +- .../src/test/java/dap4/test/TestDSP.java | 4 +- .../src/test/java/dap4/test/TestFilters.java | 347 +++++++++--------- .../src/test/java/dap4/test/TestH5Iosp.java | 19 +- .../src/test/java/dap4/test/TestNc4Iosp.java | 102 +++-- .../test/java/dap4/test/TestParserDMR.java | 9 +- .../src/test/java/dap4/test/TestSerial.java | 10 +- .../dap4/test/TestServletConstraints.java | 2 +- .../mock/web/MockTdsServletContext.java | 6 +- .../unidata/util/test/UnitTestCommon.java | 2 + 70 files changed, 1517 insertions(+), 675 deletions(-) create mode 100644 dap4/d4tests/src/test/data/resources/TestDSP/baseline/test_ncml.ncml.raw.txt create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dap create mode 100644 dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dmr create mode 100644 dap4/d4tests/src/test/data/resources/TestParsers/baseline/test_ncml.ncml.dmp diff --git a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java index a03f378ff7..39d6a8cf42 100644 --- a/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java +++ b/dap4/d4cdm/src/main/java/dap4/cdm/nc2/DapNetcdfFile.java @@ -100,6 +100,7 @@ public void setProgress(String msg, int progress) * @param location URL for the request. Note that if this is * intended to send to a file oriented * DSP, then if must be converted to an absolute path. + * Connection URLs are assumed to not be % escaped. * @param cancelTask check if task is cancelled; may be null. * @throws IOException */ @@ -123,7 +124,7 @@ public DapNetcdfFile(String location, CancelTask cancelTask) } else { // Not a file url this.dsplocation = xuri.assemble(XURI.URLBASE); try { - this.dsp = new FileDSP().open(new URI(this.dsplocation)); + this.dsp = new HttpDSP().open(new URI(this.dsplocation)); } catch (URISyntaxException e) { throw new DapException(e); } diff --git a/dap4/d4core/src/main/java/dap4/core/util/Escape.java b/dap4/d4core/src/main/java/dap4/core/util/Escape.java index 1fd8dc0d80..29eefa437a 100644 --- a/dap4/d4core/src/main/java/dap4/core/util/Escape.java +++ b/dap4/d4core/src/main/java/dap4/core/util/Escape.java @@ -41,6 +41,10 @@ public class Escape // define the default entity characters to escape static public final String ENTITYESCAPES = "\\<>&\"'"; + // (Minimal?) Set of non-alphanum characters that need to be escaped in a query + // before sending it to server Subset of nonAlphaNumeric + static final String URLESCAPECHARS = " !\"#$%'()+,/:;<>?@[]\\^_`|{}~"; + // Define the alphan characters /* Defind loose set of characters that can appear in an xml entity name */ @@ -252,6 +256,29 @@ else switch (c) { return clear.toString(); } + /** + * Search a string with respect to an unescaped haracter + * and taking backslashes into consideration. + * + * @param s The string to search + * @param cs The character for which to search + * @param start Start the search at this position + * @return index of firt occurrencce or -1 + */ + static public int + backslashindexof(String s, char cs, int start) + { + int len = s.length(); + int i = start; + while(i < len) { + char c = s.charAt(i); + if(c == cs) return i; + if(c == '\\' && i < (len - 1)) i++; + i++; + } + return -1; // not found + } + /** * Split a string with respect to a separator * character and taking backslashes into consideration. @@ -259,30 +286,27 @@ else switch (c) { * @param s The string to split * @param sep The character on which to split * @return a List of strings (all with escaping still intact) - * representing s split at unescaped instances of sep. + * representing s split at unescaped instances of sep. */ static public List backslashsplit(String s, char sep) { List path = new ArrayList(); int len = s.length(); - StringBuilder piece = new StringBuilder(); int i = 0; - for(; i <= len - 1; i++) { - char c = s.charAt(i); - if(c == '\\' && i < (len - 1)) { - piece.append(c); // keep escapes in place - piece.append(s.charAt(++i)); - } else if(c == sep) { - path.add(piece.toString()); - piece.setLength(0); - } else - piece.append(c); + int prev = -1; + while(i < len) { + int pos = backslashindexof(s,sep,i); + if(pos < 0) pos = len; + String piece = s.substring(prev+1,pos); + path.add(piece); + prev = pos; + i = pos+1; } - path.add(piece.toString()); return path; } + /* static public String backslashEscape(String s, String wrt) { StringBuilder buf = new StringBuilder(); @@ -344,11 +368,8 @@ static public String urlDecode(String s) return s; } - // (Minimal?) Set of non-alphanum characters that need to be escaped in a query - // before sending it to server - static final String URLESCAPECHARS = " %"; - static public String urlEncodeQuery(String s) + static public String urlEncode(String s) { if(s == null || s.length() == 0) return s; if(false) try {// Note that URLEncoder over encodes. For practical purposes, @@ -368,6 +389,7 @@ static public String urlEncodeQuery(String s) } else buf.append(c); } + s = buf.toString(); } return s; } diff --git a/dap4/d4lib/src/main/java/dap4/dap4lib/XURI.java b/dap4/d4lib/src/main/java/dap4/dap4lib/XURI.java index 3ac1627e7e..501b35c1d6 100644 --- a/dap4/d4lib/src/main/java/dap4/dap4lib/XURI.java +++ b/dap4/d4lib/src/main/java/dap4/dap4lib/XURI.java @@ -85,23 +85,23 @@ public XURI(String xurl) List protocols = DapUtil.getProtocols(xurl, breakpoint); // should handle drive letters also String remainder = xurl.substring(breakpoint[0], xurl.length()); switch (protocols.size()) { - case 0: // pretend it is a file - this.formatprotocol = "file"; - this.baseprotocol = "file"; - break; - case 1: - this.formatprotocol = protocols.get(0); - if("file".equalsIgnoreCase(this.formatprotocol)) - this.baseprotocol = "file"; // default conversion - else - this.baseprotocol = "http"; // default conversion - break; - case 2: - this.baseprotocol = protocols.get(0); - this.formatprotocol = protocols.get(1); - break; - default: - throw new URISyntaxException(xurl, "Too many protocols: at most 2 allowed"); + case 0: // pretend it is a file + this.formatprotocol = "file"; + this.baseprotocol = "file"; + break; + case 1: + this.formatprotocol = protocols.get(0); + if("file".equalsIgnoreCase(this.formatprotocol)) + this.baseprotocol = "file"; // default conversion + else + this.baseprotocol = "http"; // default conversion + break; + case 2: + this.baseprotocol = protocols.get(0); + this.formatprotocol = protocols.get(1); + break; + default: + throw new URISyntaxException(xurl, "Too many protocols: at most 2 allowed"); } this.isfile = "file".equals(this.baseprotocol); // The standard URI parser does not handle 'file:' very well, @@ -122,7 +122,39 @@ public XURI(String xurl) throws URISyntaxException { // Construct a usable url and parse it - URI uri = new URI(baseprotocol + ":" + remainder); + // First, break off the query and fragment parts + // because they need % escaping + int len = remainder.length(); + int qindex = Escape.backslashindexof(remainder, '?', 0); + int findex = Escape.backslashindexof(remainder, '#', 0); + int min = -1; // min pos of query or frag + if(qindex >= 0) min = qindex; + else if(findex >= 0) min = findex; + else min = len; + if(qindex >= 0 && findex >= 0 && qindex < findex) + findex = Escape.backslashindexof(remainder, '#', qindex + 1); + String frag = null; + String query = null; + if(qindex >= 0) + query = remainder.substring(qindex + 1, (findex >= 0 ? findex : len)); // exclude the '?' + if(findex > 0) + frag = remainder.substring(findex + 1, len); // exclude the '#' + remainder = remainder.substring(0, min); + frag = Escape.urlEncode(frag); + query = Escape.urlEncode(query); + StringBuilder buf = new StringBuilder(); + buf.append(baseprotocol); + buf.append(':'); + buf.append(remainder); + if(query != null) { + buf.append('?'); + buf.append(query); + } + if(frag != null) { + buf.append('#'); + buf.append(frag); + } + URI uri = new URI(buf.toString()); // Extract the parts of the uri so they can // be modified and later reassembled this.userinfo = canonical(uri.getUserInfo()); @@ -294,19 +326,19 @@ public Map getFragFields() int useformat = (parts.contains(Parts.FORMAT) ? 1 : 0); int usebase = (parts.contains(Parts.BASE) ? 2 : 0); switch (useformat + usebase) { - case 0 + 0: // neither - break; - case 1 + 0: // FORMAT only - uri.append(this.formatprotocol + ":"); - break; - case 2 + 0: // BASE only - uri.append(this.baseprotocol + ":"); - break; - case 2 + 1: // both - uri.append(this.formatprotocol + ":"); - if(!this.baseprotocol.equals(this.formatprotocol)) + case 0 + 0: // neither + break; + case 1 + 0: // FORMAT only + uri.append(this.formatprotocol + ":"); + break; + case 2 + 0: // BASE only + uri.append(this.baseprotocol + ":"); + break; + case 2 + 1: // both uri.append(this.formatprotocol + ":"); - break; + if(!this.baseprotocol.equals(this.formatprotocol)) + uri.append(this.formatprotocol + ":"); + break; } uri.append(this.baseprotocol.equals("file") ? "/" : "//"); diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java index 1db3cf67e5..236bcd1613 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java @@ -130,15 +130,13 @@ public ProtocolMatch(String proto, String replace, Class dsp) } } - ////////////////////////////////////////////////// + //////////////////////////h//////////////////////// // Static variables static ExtMatch[] match = new ExtMatch[]{ - new ExtMatch(".dmr", SynDSP.class), new ExtMatch(".syn", SynDSP.class), new ExtMatch(".nc", Nc4DSP.class), new ExtMatch(".hdf5", Nc4DSP.class), - new ExtMatch(null, FileDSP.class) // default }; static FragMatch[] frags = new FragMatch[]{ @@ -168,15 +166,18 @@ static protected DSP open(NetcdfFile ncfile, DapContext cxt) static protected DSP open(File file, DapContext cxt) throws IOException { - // Choose the DSP based on the file extension + // Choose the DSP based on the file extension; + // Note that we need to ignore any trailing .dmr or .dap String path = file.getPath(); Class cldsp = null; - for(ExtMatch em : match) { - if(em.ext == null) { - cldsp = em.dsp; - break; + String core = filepathcore(path); + if(core != null && core.length() > 0) { + for(ExtMatch em : match) { + if(core.endsWith(em.ext)) { + cldsp = em.dsp; + break; + } } - if(path.endsWith(em.ext)) {cldsp = em.dsp; break;} } if(cldsp == null) throw new DapException("Indeciperable file: " + file); @@ -234,7 +235,7 @@ static protected DSP open(URI uri, DapContext cxt) /**************************************************/ // Exported versions for NetcdfFile and String - static public synchronized DSP open(DapRequest drq, NetcdfFile ncfile, DapContext cxt) + public static synchronized DSP open(DapRequest drq, NetcdfFile ncfile, DapContext cxt) throws IOException { assert cxt != null && ncfile != null; @@ -243,7 +244,7 @@ static public synchronized DSP open(DapRequest drq, NetcdfFile ncfile, DapContex return dsp; } - static public synchronized DSP open(DapRequest drq, String target, DapContext cxt) + public static synchronized DSP open(DapRequest drq, String target, DapContext cxt) throws IOException { assert cxt != null && target != null; @@ -266,11 +267,13 @@ static public synchronized DSP open(DapRequest drq, String target, DapContext cx path = target; } + String core = filepathcore(path); // remove any trailing .dmr|.dap + if(dsp == null) { // See if this can open as a NetcdfFile|NetcdfDataset NetcdfFile ncfile = null; try { - ncfile = drq.getController().getNetcdfFile(drq,path); + ncfile = drq.getController().getNetcdfFile(drq,core); } catch (IOException ioe) { ncfile = null; } @@ -281,7 +284,7 @@ static public synchronized DSP open(DapRequest drq, String target, DapContext cx if(dsp == null) { // Finally, try to open as a some kind of File object - File file = new File(path); + File file = new File(core); // Complain if it does not exist if(!file.exists()) throw new DapException("Not found: " + target) @@ -311,5 +314,15 @@ static public synchronized DSP open(DapRequest drq, String target, DapContext cx return map; } + // Given a path that might end in .dmr|.dap, strip it of that ending + // to get what should be the essential file path + static String filepathcore(String path) + { + if(path == null) return path; + int cut = 0; + if(path.endsWith(".dmr")) cut = ".dmr".length(); + else if(path.endsWith(".dap")) cut = ".dap".length(); + return path.substring(0,path.length() - cut); + } -} // DapCache +} diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java index 6cc4740087..580530d6f5 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapRequest.java @@ -101,9 +101,9 @@ public DapRequest(DapController controller, } catch (MalformedURLException e) { this.resourceroot = null; } - if(this.resourceroot == null) - throw new DapException("Cannot locate resource root"); } + if(this.resourceroot == null) + throw new DapException("Cannot locate resource root"); try { parse(); } catch (IOException ioe) { diff --git a/dap4/d4tests/src/test/data/resources/TestDSP/baseline/test_ncml.ncml.raw.txt b/dap4/d4tests/src/test/data/resources/TestDSP/baseline/test_ncml.ncml.raw.txt new file mode 100644 index 0000000000..96a300897c --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestDSP/baseline/test_ncml.ncml.raw.txt @@ -0,0 +1,20 @@ + + + + + + + + + + +/t = 17 +/uv8 = 240 +/v16 = 32700 +/uv32 = 111000 + diff --git a/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.1.dap b/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.1.dap index f36a08093f..278ec1a3d9 100644 --- a/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.1.dap +++ b/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.1.dap @@ -1,3 +1,57 @@ -count=5 - -1169720286 18719 -2088732436 -16591 -1123468835 -18686 2125143125 -21899 1268468519 -22144 - checksum = 4aad97d4 +netcdf test_sequence_1.syn.dap.1 { + variables: + + Sequence { + int i1; + short sh1; + } s(*); + string s:_DAP4_Checksum_CRC32 = "0x9a5208e5"; + + + // global attributes: + string :_dap4.ce = "/s"; + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + s = + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s +} diff --git a/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.2.dap b/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.2.dap index 7d7d18b90f..3ad80a9d57 100644 --- a/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.2.dap +++ b/dap4/d4tests/src/test/data/resources/TestFilters/baseline/test_sequence_1.syn.2.dap @@ -1,3 +1,57 @@ -count=3 - -1169720286 18719 -2088732436 -16591 -1123468835 -18686 - checksum = 6a9c6608 +netcdf test_sequence_1.syn.dap.2 { + variables: + + Sequence { + int i1; + short sh1; + } s(*); + string s:_DAP4_Checksum_CRC32 = "0x9a5208e5"; + + + // global attributes: + string :_dap4.ce = "/s|i1<0"; + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + s = + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s + { + i1 = + {-1169720286, -2088732436, -1123468835, 2125143125, 1268468519 + } + sh1 = + {18719, -16591, -18686, -21899, -22144 + } + } s +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dap new file mode 100644 index 0000000000..dccc7f15a7 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dap @@ -0,0 +1,57 @@ +netcdf test_atomic_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d1 = 1; + d2 = 2; + d3 = 3; + d4 = 4; + d5 = 5; + variables: + ubyte vu8(d2, d3); + + short v16(d4); + + uint vu32(d2, d3); + + double vd(d2); + + char vc(d2); + + String vs(d2, d2); + + opaque vo(d1, d2); + + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + vu8 = + { + {255, 1, 2}, + {3, 4, 5} + } + v16 = + {1, 2, 3, 4} + vu32 = + { + {5, 4, 3}, + {2, 1, 0} + } + vd = + {17.9, 1024.8} + vc = "@&" + vs = + { "hello world", " +", "Καλημέα", "abc" + } + vo = 0x0123456789abcdef, + 0xabcdef0000000000; + + primary_cloud = + {0, 2, 0, 1, 127} +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dmr new file mode 100644 index 0000000000..eaf85efa89 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.hdf5.dmr @@ -0,0 +1,31 @@ +netcdf test_atomic_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d1 = 1; + d2 = 2; + d3 = 3; + d4 = 4; + d5 = 5; + variables: + ubyte vu8(d2, d3); + + short v16(d4); + + uint vu32(d2, d3); + + double vd(d2); + + char vc(d2); + + String vs(d2, d2); + + opaque vo(d1, d2); + + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dap index a190c8fd5c..dccc7f15a7 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dap @@ -1,56 +1,57 @@ -netcdf test_atomic_array { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - dimensions: - d1 = 1; - d2 = 2; - d3 = 3; - d4 = 4; - d5 = 5; - variables: - ubyte vu8(d2, d3); - - short v16(d4); - - uint vu32(d2, d3); - - double vd(d2); - - char vc(d2); - - String vs(d2, d2); - - opaque vo(d1, d2); - - enum cloud_class_t primary_cloud(d5); - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -vu8 = - { - {255, 1, 2}, - {3, 4, 5} - } -v16 = - {1, 2, 3, 4} -vu32 = - { - {5, 4, 3}, - {2, 1, 0} - } -vd = - {17.9, 1024.8} -vc = "@&" -vs = - { "hello world", " +netcdf test_atomic_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d1 = 1; + d2 = 2; + d3 = 3; + d4 = 4; + d5 = 5; + variables: + ubyte vu8(d2, d3); + + short v16(d4); + + uint vu32(d2, d3); + + double vd(d2); + + char vc(d2); + + String vs(d2, d2); + + opaque vo(d1, d2); + + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + vu8 = + { + {255, 1, 2}, + {3, 4, 5} + } + v16 = + {1, 2, 3, 4} + vu32 = + { + {5, 4, 3}, + {2, 1, 0} + } + vd = + {17.9, 1024.8} + vc = "@&" + vs = + { "hello world", " ", "Καλημέα", "abc" - } -vo = 0x0123456789abcdef, - 0xabcdef0000000000; + } + vo = 0x0123456789abcdef, + 0xabcdef0000000000; -primary_cloud = - {0, 2, 0, 1, 127} + primary_cloud = + {0, 2, 0, 1, 127} } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dmr index 822a30cbc9..eaf85efa89 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dmr +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_array.nc.nc4.dmr @@ -1,31 +1,31 @@ -netcdf test_atomic_array { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - dimensions: - d1 = 1; - d2 = 2; - d3 = 3; - d4 = 4; - d5 = 5; - variables: - ubyte vu8(d2, d3); - - short v16(d4); - - uint vu32(d2, d3); - - double vd(d2); - - char vc(d2); - - String vs(d2, d2); - - opaque vo(d1, d2); - - enum cloud_class_t primary_cloud(d5); - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +netcdf test_atomic_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d1 = 1; + d2 = 2; + d3 = 3; + d4 = 4; + d5 = 5; + variables: + ubyte vu8(d2, d3); + + short v16(d4); + + uint vu32(d2, d3); + + double vd(d2); + + char vc(d2); + + String vs(d2, d2); + + opaque vo(d1, d2); + + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dap new file mode 100644 index 0000000000..fd14dbf7a9 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dap @@ -0,0 +1,58 @@ +netcdf test_atomic_types { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + byte v8; + + ubyte vu8; + + short v16; + + ushort vu16; + + int v32; + + uint vu32; + + long v64; + + ulong vu64; + + float vf; + + double vd; + + char vc; + + String vs; + + opaque vo; + + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + enum secondary_cloud secondary_cloud; + string secondary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + v8 = -128 + vu8 = 255 + v16 = -32768 + vu16 = 65535 + v32 = 2147483647 + vu32 = 4294967295 + v64 = 9223372036854775807 + vu64 = 18446744073709551615 + vf = 3.1415927 + vd = 3.141592653589793 + vc = @ + vs = "hello world" + vo = 0x0123456789abcdef; + + primary_cloud = 2 + secondary_cloud = 127 +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dmr new file mode 100644 index 0000000000..097d75fb9f --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.hdf5.dmr @@ -0,0 +1,40 @@ +netcdf test_atomic_types { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + byte v8; + + ubyte vu8; + + short v16; + + ushort vu16; + + int v32; + + uint vu32; + + long v64; + + ulong vu64; + + float vf; + + double vd; + + char vc; + + String vs; + + opaque vo; + + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + enum secondary_cloud secondary_cloud; + string secondary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dap index 205e48eb6d..fd14dbf7a9 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dap @@ -1,57 +1,58 @@ -netcdf test_atomic_types { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - variables: - byte v8; - - ubyte vu8; - - short v16; - - ushort vu16; - - int v32; - - uint vu32; - - long v64; - - ulong vu64; - - float vf; - - double vd; - - char vc; - - String vs; - - opaque vo; - - enum cloud_class_t primary_cloud; - string primary_cloud:_FillValue = "Missing"; - - enum cloud_class_t secondary_cloud; - string secondary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -v8 =-128 -vu8 =255 -v16 =-32768 -vu16 =65535 -v32 =2147483647 -vu32 =4294967295 -v64 =9223372036854775807 -vu64 =18446744073709551615 -vf =3.1415927 -vd =3.141592653589793 -vc =@ -vs = "hello world" -vo = 0x0123456789abcdef; - -primary_cloud =2 -secondary_cloud =127 +netcdf test_atomic_types { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + byte v8; + + ubyte vu8; + + short v16; + + ushort vu16; + + int v32; + + uint vu32; + + long v64; + + ulong vu64; + + float vf; + + double vd; + + char vc; + + String vs; + + opaque vo; + + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + enum secondary_cloud secondary_cloud; + string secondary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + v8 = -128 + vu8 = 255 + v16 = -32768 + vu16 = 65535 + v32 = 2147483647 + vu32 = 4294967295 + v64 = 9223372036854775807 + vu64 = 18446744073709551615 + vf = 3.1415927 + vd = 3.141592653589793 + vc = @ + vs = "hello world" + vo = 0x0123456789abcdef; + + primary_cloud = 2 + secondary_cloud = 127 } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dmr index d771557b7f..097d75fb9f 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dmr +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_atomic_types.nc.nc4.dmr @@ -1,40 +1,40 @@ -netcdf test_atomic_types { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - variables: - byte v8; - - ubyte vu8; - - short v16; - - ushort vu16; - - int v32; - - uint vu32; - - long v64; - - ulong vu64; - - float vf; - - double vd; - - char vc; - - String vs; - - opaque vo; - - enum cloud_class_t primary_cloud; - string primary_cloud:_FillValue = "Missing"; - - enum cloud_class_t secondary_cloud; - string secondary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +netcdf test_atomic_types { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + byte v8; + + ubyte vu8; + + short v16; + + ushort vu16; + + int v32; + + uint vu32; + + long v64; + + ulong vu64; + + float vf; + + double vd; + + char vc; + + String vs; + + opaque vo; + + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + enum secondary_cloud secondary_cloud; + string secondary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dap new file mode 100644 index 0000000000..9238c2763d --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dap @@ -0,0 +1,14 @@ +netcdf test_enum { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + primary_cloud = 2 +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dmr new file mode 100644 index 0000000000..db22e48ada --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.hdf5.dmr @@ -0,0 +1,11 @@ +netcdf test_enum { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dap index 7723b215c7..9238c2763d 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dap @@ -1,13 +1,14 @@ -netcdf test_enum { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - variables: - enum cloud_class_t primary_cloud; - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -primary_cloud =2 +netcdf test_enum { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + primary_cloud = 2 } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dmr index 0b98070c42..db22e48ada 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dmr +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum.nc.nc4.dmr @@ -1,11 +1,11 @@ -netcdf test_enum { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - variables: - enum cloud_class_t primary_cloud; - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +netcdf test_enum { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + variables: + enum primary_cloud primary_cloud; + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dap new file mode 100644 index 0000000000..3caa74dbe6 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dap @@ -0,0 +1,17 @@ +netcdf test_enum_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d5 = 5; + variables: + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + primary_cloud = + {0, 2, 0, 1, 127} +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dmr new file mode 100644 index 0000000000..76c2181ccc --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.hdf5.dmr @@ -0,0 +1,13 @@ +netcdf test_enum_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d5 = 5; + variables: + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dap index 6bc8a7ac6d..3caa74dbe6 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dap @@ -1,16 +1,17 @@ -netcdf test_enum_array { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - dimensions: - d5 = 5; - variables: - enum cloud_class_t primary_cloud(d5); - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -primary_cloud = - {0, 2, 0, 1, 127} +netcdf test_enum_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d5 = 5; + variables: + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + primary_cloud = + {0, 2, 0, 1, 127} } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dmr index 0ba318f4b9..76c2181ccc 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dmr +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_enum_array.nc.nc4.dmr @@ -1,13 +1,13 @@ -netcdf test_enum_array { - types: - byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; - - dimensions: - d5 = 5; - variables: - enum cloud_class_t primary_cloud(d5); - string primary_cloud:_FillValue = "Missing"; - - // global attributes: - string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +netcdf test_enum_array { + types: + byte enum cloud_class_t { Clear = 0, Cumulonimbus = 1, Stratus = 2, Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127}; + + dimensions: + d5 = 5; + variables: + enum primary_cloud primary_cloud(d5); + string primary_cloud:_FillValue = "Missing"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dap new file mode 100644 index 0000000000..2dd4ba882b --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dap @@ -0,0 +1,10 @@ +netcdf test_one_var { + variables: + int t; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + t = 17 +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dmr new file mode 100644 index 0000000000..e0de669035 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.hdf5.dmr @@ -0,0 +1,7 @@ +netcdf test_one_var { + variables: + int t; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.nc4.dap index 0e0abd2601..2dd4ba882b 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_var.nc.nc4.dap @@ -4,6 +4,7 @@ netcdf test_one_var { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -t =17 + + data: + t = 17 } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dap new file mode 100644 index 0000000000..55b21c6b0b --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dap @@ -0,0 +1,13 @@ +netcdf test_one_vararray { + dimensions: + d2 = 2; + variables: + int t(d2); + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + t = + {17, 37} +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dmr new file mode 100644 index 0000000000..e2854e0bef --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.hdf5.dmr @@ -0,0 +1,9 @@ +netcdf test_one_vararray { + dimensions: + d2 = 2; + variables: + int t(d2); + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.nc4.dap index 8c531d9081..55b21c6b0b 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_one_vararray.nc.nc4.dap @@ -6,7 +6,8 @@ netcdf test_one_vararray { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -t = - {17, 37} + + data: + t = + {17, 37} } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dap new file mode 100644 index 0000000000..a7ea050ebb --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dap @@ -0,0 +1,68 @@ +netcdf test_struct_array { + dimensions: + dx = 4; + dy = 3; + variables: + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } s(dx, dy); + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + s = + { + x = 1 + y = -1 + } s(0) + { + x = 17 + y = 37 + } s(1) + { + x = -32767 + y = 32767 + } s(2) + { + x = -1 + y = 3 + } s(3) + { + x = -2 + y = 2 + } s(4) + { + x = -3 + y = 1 + } s(5) + { + x = -4 + y = 12 + } s(6) + { + x = -8 + y = 8 + } s(7) + { + x = -12 + y = 4 + } s(8) + { + x = -5 + y = 15 + } s(9) + { + x = -10 + y = 10 + } s(10) + { + x = -15 + y = 5 + } s(11) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dmr new file mode 100644 index 0000000000..da24f7ca51 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.hdf5.dmr @@ -0,0 +1,17 @@ +netcdf test_struct_array { + dimensions: + dx = 4; + dy = 3; + variables: + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } s(dx, dy); + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.nc4.dap index 8e016581fc..a7ea050ebb 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_array.nc.nc4.dap @@ -14,54 +14,55 @@ netcdf test_struct_array { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -s = - { - x =1 - y =-1 - } s(0) - { - x =17 - y =37 - } s(1) - { - x =-32767 - y =32767 - } s(2) - { - x =-1 - y =3 - } s(3) - { - x =-2 - y =2 - } s(4) - { - x =-3 - y =1 - } s(5) - { - x =-4 - y =12 - } s(6) - { - x =-8 - y =8 - } s(7) - { - x =-12 - y =4 - } s(8) - { - x =-5 - y =15 - } s(9) - { - x =-10 - y =10 - } s(10) - { - x =-15 - y =5 - } s(11) + + data: + s = + { + x = 1 + y = -1 + } s(0) + { + x = 17 + y = 37 + } s(1) + { + x = -32767 + y = 32767 + } s(2) + { + x = -1 + y = 3 + } s(3) + { + x = -2 + y = 2 + } s(4) + { + x = -3 + y = 1 + } s(5) + { + x = -4 + y = 12 + } s(6) + { + x = -8 + y = 8 + } s(7) + { + x = -12 + y = 4 + } s(8) + { + x = -5 + y = 15 + } s(9) + { + x = -10 + y = 10 + } s(10) + { + x = -15 + y = 5 + } s(11) } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dap new file mode 100644 index 0000000000..b107bec63b --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dap @@ -0,0 +1,39 @@ +netcdf test_struct_nested { + variables: + + Structure { + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } field1; + + + Structure { + int x; + int y; + } field2; + + } x; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + x = + { + field1 = + { + x = 1 + y = -2 + } field1(0) + field2 = + { + x = 255 + y = 90 + } field2(0) + } x(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dmr new file mode 100644 index 0000000000..5c6dbc9f0e --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.hdf5.dmr @@ -0,0 +1,24 @@ +netcdf test_struct_nested { + variables: + + Structure { + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } field1; + + + Structure { + int x; + int y; + } field2; + + } x; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.nc4.dap index 910c4da5be..b107bec63b 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_nested.nc.nc4.dap @@ -21,18 +21,19 @@ netcdf test_struct_nested { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + data: - x = + x = { - field1 = + field1 = { - x = 1 - y = -2 - } x.field1(0) - field2 = + x = 1 + y = -2 + } field1(0) + field2 = { - x = 255 - y = 90 - } x.field2(0) - } x(0) -} + x = 255 + y = 90 + } field2(0) + } x(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dap new file mode 100644 index 0000000000..7d3ec162e6 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dap @@ -0,0 +1,21 @@ +netcdf test_struct_type { + variables: + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } s; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + s = + { + x = 1 + y = -2 + } s(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dmr new file mode 100644 index 0000000000..b9ec67883a --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.hdf5.dmr @@ -0,0 +1,14 @@ +netcdf test_struct_type { + variables: + + Structure { + int x; + string x:_CoordinateAxisType = "GeoX"; + int y; + string y:_CoordinateAxisType = "GeoY"; + } s; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.nc4.dap index 4992d822d1..7d3ec162e6 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_struct_type.nc.nc4.dap @@ -11,11 +11,11 @@ netcdf test_struct_type { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + data: - s = + s = { - x = 1 - y = -2 - } s(0) -} - + x = 1 + y = -2 + } s(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dap new file mode 100644 index 0000000000..74dec81c00 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dap @@ -0,0 +1,12 @@ +netcdf test_vlen1 { + variables: + int x(*); + string x:_CoordinateAxisType = "GeoX"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + x = + {1, 3, 5, 7} +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dmr new file mode 100644 index 0000000000..998c2beeeb --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.hdf5.dmr @@ -0,0 +1,8 @@ +netcdf test_vlen1 { + variables: + int x(*); + string x:_CoordinateAxisType = "GeoX"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.nc4.dap index 5d305d252d..74dec81c00 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen1.nc.nc4.dap @@ -5,7 +5,8 @@ netcdf test_vlen1 { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -x = - {1, 3, 5, 7} + + data: + x = + {1, 3, 5, 7} } diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dap new file mode 100644 index 0000000000..5bbee64fda --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dap @@ -0,0 +1,22 @@ +netcdf test_vlen2 { + dimensions: + d3 = 3; + d2 = 2; + variables: + int x(d3, d2, *); + string x:_CoordinateAxisType = "GeoX"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + x = + { + {1, 3, 5, 7}, + {100, 200}, + {-1, -2}, + {1, 3, 5, 7}, + {100, 200}, + {-1, -2} + } +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dmr new file mode 100644 index 0000000000..3c5a459470 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.hdf5.dmr @@ -0,0 +1,11 @@ +netcdf test_vlen2 { + dimensions: + d3 = 3; + d2 = 2; + variables: + int x(d3, d2, *); + string x:_CoordinateAxisType = "GeoX"; + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.nc4.dap index d3d3a93f76..5bbee64fda 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen2.nc.nc4.dap @@ -8,14 +8,15 @@ netcdf test_vlen2 { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + data: - x = - { - {1, 3, 5, 7}, - {100, 200}, - {-1, -2}, - {1, 3, 5, 7}, - {100, 200}, - {-1, -2} - } -} + x = + { + {1, 3, 5, 7}, + {100, 200}, + {-1, -2}, + {1, 3, 5, 7}, + {100, 200}, + {-1, -2} + } +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dap new file mode 100644 index 0000000000..bf28e91446 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dap @@ -0,0 +1,18 @@ +netcdf test_vlen3 { + variables: + + Structure { + int f1(*); + } v1; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + v1 = + { + f1 = + {1, 3, 5, 7} + } v1(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dmr new file mode 100644 index 0000000000..348943c043 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.hdf5.dmr @@ -0,0 +1,11 @@ +netcdf test_vlen3 { + variables: + + Structure { + int f1(*); + } v1; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.nc4.dap index 4378f59fec..bf28e91446 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen3.nc.nc4.dap @@ -8,10 +8,11 @@ netcdf test_vlen3 { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + data: - v1 = + v1 = { - f1 = - {1, 3, 5, 7} - } v1(0) -} + f1 = + {1, 3, 5, 7} + } v1(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dap new file mode 100644 index 0000000000..67d348e94e --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dap @@ -0,0 +1,21 @@ +netcdf test_vlen4 { + variables: + + Structure { + int f1(2, *); + } v1; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + v1 = + { + f1 = + { + {1, 3, 5, 7}, + {100, 200} + } + } v1(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dmr new file mode 100644 index 0000000000..066d3c44e6 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.hdf5.dmr @@ -0,0 +1,11 @@ +netcdf test_vlen4 { + variables: + + Structure { + int f1(2, *); + } v1; + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.nc4.dap index 98123a9714..67d348e94e 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen4.nc.nc4.dap @@ -8,13 +8,14 @@ netcdf test_vlen4 { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + data: - v1 = + v1 = { - f1 = - { - {1, 3, 5, 7}, - {100, 200} - } - } v1(0) -} + f1 = + { + {1, 3, 5, 7}, + {100, 200} + } + } v1(0) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dap new file mode 100644 index 0000000000..b4e6b4c25b --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dap @@ -0,0 +1,24 @@ +netcdf test_vlen5 { + dimensions: + d2 = 2; + variables: + + Structure { + int v(*); + } v1(d2); + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; + + data: + v1 = + { + v = + {1, 3, 5, 7} + } v1(0) + { + v = + {100, 200} + } v1(1) +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dmr b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dmr new file mode 100644 index 0000000000..057180f582 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.hdf5.dmr @@ -0,0 +1,13 @@ +netcdf test_vlen5 { + dimensions: + d2 = 2; + variables: + + Structure { + int v(*); + } v1(d2); + + + // global attributes: + string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; +} diff --git a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.nc4.dap b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.nc4.dap index c81542fb0b..b4e6b4c25b 100644 --- a/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.nc4.dap +++ b/dap4/d4tests/src/test/data/resources/TestIosp/baseline/test_vlen5.nc.nc4.dap @@ -10,14 +10,15 @@ netcdf test_vlen5 { // global attributes: string :_CoordSysBuilder = "ucar.nc2.dataset.conv.DefaultConvention"; - data: -v1 = - { - v = - {1, 3, 5, 7} - } v1(0) - { - v = - {100, 200} - } v1(1) + + data: + v1 = + { + v = + {1, 3, 5, 7} + } v1(0) + { + v = + {100, 200} + } v1(1) } diff --git a/dap4/d4tests/src/test/data/resources/TestParsers/baseline/test_ncml.ncml.dmp b/dap4/d4tests/src/test/data/resources/TestParsers/baseline/test_ncml.ncml.dmp new file mode 100644 index 0000000000..bb933bcffd --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestParsers/baseline/test_ncml.ncml.dmp @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_atomic_types.nc.dmr b/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_atomic_types.nc.dmr index 88340335c2..e4d028fb89 100644 --- a/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_atomic_types.nc.dmr +++ b/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_atomic_types.nc.dmr @@ -19,8 +19,8 @@ - - + + @@ -36,9 +36,9 @@ - + - + diff --git a/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_numeric_types.nc.dmr b/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_numeric_types.nc.dmr index 57cb8ca453..29b6651546 100644 --- a/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_numeric_types.nc.dmr +++ b/dap4/d4tests/src/test/data/resources/testfiles/dmr/test_numeric_types.nc.dmr @@ -5,8 +5,8 @@ xmlns="http://xml.opendap.org/ns/DAP/4.0#" xmlns:dap="http://xml.opendap.org/ns/DAP/4.0#"> - - + + diff --git a/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java b/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java index e6e6bda691..65fadf3d11 100644 --- a/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java +++ b/dap4/d4tests/src/test/java/dap4/test/DapTestCommon.java @@ -68,6 +68,8 @@ abstract public class DapTestCommon extends UnitTestCommon static final String D4TESTDIRNAME = "d4tests"; + static final String DAP4MODE = "#mode=dap4"; + // Equivalent to the path to the webapp/d4ts for testing purposes static protected final String DFALTRESOURCEPATH = "/src/test/data/resources"; static protected Class NC4IOSP = ucar.nc2.jni.netcdf.Nc4Iosp.class; @@ -103,9 +105,10 @@ public Mocker(String servletname, String url, DapController controller, DapTestC // There appears to be bug in the spring core.io code // such that it assumes absolute paths start with '/'. // So, check for windows drive and prepend 'file:/' as a hack. - if(DapUtil.hasDriveLetter(testdir)) - testdir = "/" + testdir; - testdir = "file:" + testdir; + if(DapUtil.hasDriveLetter(testdir)) { + //testdir = "/" + testdir; + testdir = "file://" + testdir; + } this.context = new MockServletContext(testdir); URI u = HTTPUtil.parseToURI(url); this.req = new MockHttpServletRequest(this.context, "GET", u.getPath()); @@ -322,9 +325,9 @@ public boolean accept(File file) ////////////////////////////////////////////////// // Static variables - static protected String dap4root = null; - static protected String dap4testroot = null; - static protected String dap4resourcedir = null; + protected static String dap4root = null; + protected static String dap4testroot = null; + protected static String dap4resourcedir = null; static { dap4root = locateDAP4Root(threddsroot); @@ -439,6 +442,7 @@ public String getTitle() public void visual(String header, String captured) { + if(captured == null) captured = ""; if(!captured.endsWith("\n")) captured = captured + "\n"; // Dump the output for visual comparison diff --git a/dap4/d4tests/src/test/java/dap4/test/GenerateRaw.java b/dap4/d4tests/src/test/java/dap4/test/GenerateRaw.java index 8973a3b06d..3939e05d8c 100644 --- a/dap4/d4tests/src/test/java/dap4/test/GenerateRaw.java +++ b/dap4/d4tests/src/test/java/dap4/test/GenerateRaw.java @@ -146,7 +146,7 @@ public TestCase(int id, String dataset, String ce) dataset = dataset.substring(0, index); } this.dataset = dataset; - this.ce = (ce == null ? null : Escape.urlEncodeQuery(ce)); + this.ce = (ce == null ? null : Escape.urlEncode(ce)); this.id = id; this.bigendian = false; this.nochecksum = false; diff --git a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java index 50e8ca8a68..605463f47a 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java @@ -39,8 +39,6 @@ public class TestCDMClient extends DapTestCommon static final String BASEEXTENSION = ".txt"; static final String INPUTEXTENSION = ".raw"; - static final String DAP4TAG = "#mode=dap4"; - static final String DATADIR = "src/test/data/resources"; // relative to dap4 root static final String BASELINEDIR = "TestCDMClient/baseline"; static final String TESTCDMINPUT = "TestCDMClient/testinput"; @@ -109,7 +107,7 @@ static String getRoot() public String getURL() { - return this.url + DAP4TAG; + return this.url + DAP4MODE; } public String getPath() diff --git a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java index ed4dc2d7a6..091e351ff9 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java @@ -29,7 +29,7 @@ public class TestConstraints extends DapTestCommon static final boolean DEBUG = false; - static final public boolean DEBUGSERVER = true; + public static final boolean DEBUGSERVER = true; ////////////////////////////////////////////////// // Constants @@ -39,8 +39,6 @@ public class TestConstraints extends DapTestCommon static final String BASEEXTENSION = "raw.txt"; static final String TESTEXTENSION = ".raw"; - static final String DAP4TAG = "protocol=dap4"; - static protected final String SERVLETPATH = "d4ts"; static protected final String RESOURCEPATH = "/src/test/data/resources"; static protected final String TESTINPUTPATH = "/testfiles"; @@ -94,7 +92,7 @@ static class TestCase String makeurl() { StringBuilder url = new StringBuilder(); - url.append("dap4://"); + url.append("http://"); url.append(server); url.append("/"); url.append(servletpath); @@ -107,8 +105,7 @@ String makeurl() url.append("="); url.append(constraint); } - url.append("#"); - url.append(DAP4TAG); + url.append(DAP4MODE); return url.toString(); } diff --git a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java index e1b9d70f12..88429128e7 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java @@ -45,7 +45,7 @@ public class TestDSP extends DapTestCommon private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static final boolean DEBUG = false; - static final boolean SHOWTESTCASES = true; + static final boolean SHOWTESTCASES = false; static final String BASEEXTENSION = "txt"; @@ -243,7 +243,7 @@ public void setup() throws Exception chooseTestcases() { if(false) { - chosentests = locate("file:", "test_struct_nested3.hdf5.raw"); + chosentests = locate("file:", "test_groups1.nc.raw"); prop_visual = true; prop_baseline = false; } else { diff --git a/dap4/d4tests/src/test/java/dap4/test/TestFilters.java b/dap4/d4tests/src/test/java/dap4/test/TestFilters.java index 80d2b60bd0..907ac1fef7 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestFilters.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestFilters.java @@ -3,15 +3,20 @@ import dap4.dap4lib.ChunkInputStream; import dap4.core.util.*; import dap4.dap4lib.RequestMode; +import dap4.dap4lib.XURI; import dap4.servlet.Generator; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ucar.httpservices.HTTPMethod; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.unidata.util.test.TestDir; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -29,91 +34,99 @@ public class TestFilters extends DapTestCommon ////////////////////////////////////////////////// // Constants + static boolean DEBUGSERVER = true; + static final boolean NCDUMP = true; // Use NcDumpW instead of D4Print - static String DATADIR = "src/test/data"; // relative to dap4 root - static String TESTDATADIR = DATADIR + "/resources/"; - static String BASELINEDIR = DATADIR + "/resources/TestFilters/baseline"; - static String TESTINPUTDIR = DATADIR + "/resources/testfiles"; - - // constants for Fake Request - static String FAKEURLPREFIX = "http://localhost:8080/d4ts"; + static protected final String SERVLETPATH = "d4ts"; + static protected final String RESOURCEPATH = "/src/test/data/resources"; + static protected final String TESTINPUTPATH = "/testfiles"; + static protected final String BASELINEDIR = "/TestFilters/baseline"; static final BigInteger MASK = new BigInteger("FFFFFFFFFFFFFFFF", 16); - static protected final int DEFAULTROWCOUNT = 5; + static final int DEFAULTROWCOUNT = 5; ////////////////////////////////////////////////// // Type Declarations - static class ConstraintTest + static class TestCase { - static String root = null; - - static ConstraintTest[] alltests; + static String servletpath = null; + static String baselinedir = null; + static String server = null; - static { - alltests = new ConstraintTest[2048]; - Arrays.fill(alltests, null); + static public void + setRoots(String servletpath, String baselinedir, String server) + { + TestCase.baselinedir = baselinedir; + TestCase.servletpath = servletpath; + TestCase.server = server; } + static boolean[] idcheck = new boolean[2048]; + + ////////////////////////////////////////////////// String title; String dataset; String constraint; - boolean xfail; - Dump.Commands template; - String testinputpath; - String baselinepath; int id; + boolean xfail; int rowcount = DEFAULTROWCOUNT; - ConstraintTest(int id, String dataset, String ce) - { - this(id, dataset, 0, ce, null, true); - } - - ConstraintTest(int id, String dataset, String ce, - Dump.Commands template) - { - this(id, dataset, 0, ce, template, false); - } - - ConstraintTest(int id, String dataset, int rows, String ce, - Dump.Commands template) + TestCase(int id, String dataset, String ce) { - this(id, dataset, rows, ce, template, false); + this(id, dataset, 0, ce, false); } - ConstraintTest(int id, String dataset, int rows, String ce, - Dump.Commands template, boolean xfail) + TestCase(int id, String dataset, int rows, String ce, boolean xfail) { - if(alltests[id] != null) + if(idcheck[id]) throw new IllegalStateException("two tests with same id"); + if(ce != null && ce.length() == 0) + ce = null; this.id = id; + idcheck[id] = true; this.title = dataset + (ce == null ? "" : ("?" + ce)); this.dataset = dataset; this.constraint = ce; this.xfail = xfail; - this.template = template; - this.testinputpath - = root + "/" + TESTINPUTDIR + "/" + dataset; - this.baselinepath - = root + "/" + BASELINEDIR + "/" + dataset + "." + String.valueOf(this.id); this.rowcount = rows == 0 ? DEFAULTROWCOUNT : rows; - alltests[id] = this; + } +// this.testinputpath = root + "/" + TESTINPUTDIR + "/" + dataset; +// this.baselinepath = root + "/" + BASELINEDIR + "/" + dataset + "." + String.valueOf(this.id); + + String + getbaseline(RequestMode mode) + { + return canonjoin(this.baselinedir, dataset) + "." + this.id + "." + mode.toString(); } String makeurl(RequestMode ext) { - String url = FAKEURLPREFIX + "/" + dataset; - if(ext != null) url += "." + ext.toString(); + StringBuilder url = new StringBuilder(); + url.append("http://"); + url.append(server); + url.append("/"); + url.append(servletpath); + url.append(TESTINPUTPATH); + url.append("/"); + url.append(dataset); + if(ext != null) {url.append("."); url.append(ext.toString());} if(constraint != null) { - url += "?" + CONSTRAINTTAG + "="; - String ce = constraint; + url.append("?"); + url.append(CONSTRAINTTAG); + url.append("="); // Escape it //ce = Escape.urlEncodeQuery(ce); - url += ce; + url.append(constraint); } - return url; + url.append(DAP4MODE); + return url.toString(); + } + + String makeName() + { + return this.dataset + "." + this.id; } public String toString() @@ -122,12 +135,6 @@ public String toString() } } - protected String - getTestFilesDir() - { - return TESTINPUTDIR; - } - ////////////////////////////////////////////////// // Instance variables @@ -136,21 +143,24 @@ public String toString() // Test cases - List alltestcases = new ArrayList(); - - List chosentests = new ArrayList(); - - String datasetpath = null; - - String testroot = null; + List alltestcases = new ArrayList<>(); + List chosentests = new ArrayList<>(); ////////////////////////////////////////////////// @Before public void setup() throws Exception { - this.testroot = getTestFilesDir(); - this.datasetpath = this.testroot + "/" + DATADIR; - defineAllTestcases(this.testroot); + String d4root = getDAP4Root(); + if(d4root == null) + throw new Exception("dap4 root cannot be located"); + testSetup(); + if(DEBUGSERVER) + HTTPMethod.MOCKEXECUTOR = new MockExecutor(getResourceRoot()); + TestCase.setRoots( + SERVLETPATH, + canonjoin(getResourceRoot(), BASELINEDIR), + TestDir.dap4TestServer); + defineAllTestcases(); chooseTestcases(); } @@ -161,140 +171,155 @@ public void setup() throws Exception { chooseTestcases() { if(false) { - chosentests = locate(1); + chosentests = locate(2); + prop_visual = true; + prop_baseline = true; } else { - for(ConstraintTest tc : alltestcases) + prop_baseline = true; + for(TestCase tc : alltestcases) chosentests.add(tc); } } protected void - defineAllTestcases(String root) + defineAllTestcases() { - ConstraintTest.root = root; this.alltestcases.add( - new ConstraintTest(1, "test_sequence_1.syn", "/s", - new Dump.Commands() - { - public void run(Dump printer) throws IOException - { - int count = printer.printcount(); - for(int j = 0;j < count;j++) { - printer.printvalue('S', 4); - printer.printvalue('S', 2); - } - printer.newline(); - printer.verifychecksum(); - } - })); + new TestCase(1, "test_sequence_1.syn", "/s")); this.alltestcases.add( - new ConstraintTest(2, "test_sequence_1.syn", "/s|i1<0", - new Dump.Commands() - { - public void run(Dump printer) throws IOException - { - int count = printer.printcount(); - for(int j = 0;j < count;j++) { - printer.printvalue('S', 4); - printer.printvalue('S', 2); - } - printer.newline(); - printer.verifychecksum(); - } - })); - + new TestCase(2, "test_sequence_1.syn", "/s|i1<0")); } - - ////////////////////////////////////////////////// - // Junit test methods + /////////////////////////////////////////////// @Test public void testFilters() - throws Exception + throws Exception/// + // Junit test methods { - boolean pass = true; - for(ConstraintTest testcase : chosentests) { - if(!doOneTest(testcase)) - pass = false; + for(TestCase testcase : chosentests) { + doOneTest(testcase); } - Assert.assertTrue("***Fail: TestServletConstraints", pass); } ////////////////////////////////////////////////// // Primary test method - boolean - doOneTest(ConstraintTest testcase) + void + doOneTest(TestCase testcase) throws Exception { - boolean pass = true; - System.out.println("Testcase: " + testcase.toString()); - Generator.setRowCount(testcase.rowcount); - pass = dodata(testcase); - return pass; - } + System.err.println("Testcase: " + testcase.toString()); - boolean - dodata(ConstraintTest testcase) - throws Exception - { - boolean pass = true; - String baseline; - RequestMode mode = RequestMode.DAP; - String methodurl = testcase.makeurl(mode); + String baselinepath = testcase.getbaseline(RequestMode.DAP); - Mocker mocker = new Mocker("dap4",methodurl,this); - byte[] byteresult = null; + System.err.println("Baseline: " + baselinepath); + Generator.setRowCount(testcase.rowcount); + String url = testcase.makeurl(RequestMode.DAP); + // Make sure url is escaped + url = new XURI(url).assemble(XURI.URLALL); + NetcdfDataset ncfile = null; try { - byteresult = mocker.execute(); - } catch (Throwable t) { - t.printStackTrace(); - return false; + ncfile = openDataset(url); + } catch (Exception e) { + throw e; + } + + // Patch the ncfile to change dataset name + String datasetname = extractDatasetname(url,Integer.toString(testcase.id)); + String metadata = (NCDUMP ? ncdumpmetadata(ncfile,datasetname) : null); + String data = (NCDUMP ? ncdumpdata(ncfile,datasetname) : null); + + if(prop_visual) { + visual("DMR: " + url, metadata); + visual("DAP: " + url, data); } if(prop_debug) { ByteOrder order = (isbigendian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); - DapDump.dumpbytes(ByteBuffer.wrap(byteresult).order(order), true); } - // Setup a ChunkInputStream - ByteArrayInputStream bytestream = new ByteArrayInputStream(byteresult); + if(prop_baseline) + writefile(baselinepath, data); - ChunkInputStream reader = new ChunkInputStream(bytestream, RequestMode.DAP, ByteOrder.nativeOrder()); + if(prop_diff) { //compare with baseline + // Read the baseline file(s) + String baselinecontent = readfile(baselinepath); + System.err.println("Comparison:"); + Assert.assertTrue("***Fail", same(getTitle(), baselinecontent, data)); + } + } - String sdmr = reader.readDMR(); // Read the DMR - if(prop_visual) - visual(methodurl, sdmr); + ////////////////////////////////////////////////// + // Dump methods - Dump printer = new Dump(); - String sdata = printer.dumpdata(reader, true, reader.getRemoteByteOrder(), testcase.template); + String ncdumpmetadata(NetcdfDataset ncfile, String datasetname) + { + boolean ok = false; + String metadata = null; + StringWriter sw = new StringWriter(); + StringBuilder args = new StringBuilder("-strict"); + if(datasetname != null) { + args.append(" -datasetname "); + args.append(datasetname); + } + // Print the meta-databuffer using these args to NcdumpW + ok = false; + try { + ok = ucar.nc2.NCdumpW.print(ncfile, args.toString(), sw, null); + } catch (IOException ioe) { + ioe.printStackTrace(); + ok = false; + } + try { + sw.close(); + } catch (IOException e) { + } + ; + if(!ok) { + System.err.println("NcdumpW failed"); + } + return sw.toString(); + } - if(prop_visual) - visual(testcase.title + ".dap", sdata); + String ncdumpdata(NetcdfDataset ncfile, String datasetname) + { + boolean ok = false; + StringWriter sw = new StringWriter(); - if(prop_baseline) - writefile(testcase.baselinepath + ".dap", sdata); - - if(prop_diff) { - //compare with baseline - // Read the baseline file - System.out.println("Note Comparison:"); - String baselinecontent = readfile(testcase.baselinepath + ".dap"); - pass = same(getTitle(),baselinecontent, sdata); - System.out.println(pass ? "Pass" : "Fail"); + StringBuilder args = new StringBuilder("-strict -vall"); + if(datasetname != null) { + args.append(" -datasetname "); + args.append(datasetname); } - return pass; + // Dump the databuffer + sw = new StringWriter(); + ok = false; + try { + ok = ucar.nc2.NCdumpW.print(ncfile, args.toString(), sw, null); + } catch (IOException ioe) { + ioe.printStackTrace(); + ok = false; + } + try { + sw.close(); + } catch (IOException e) { + } + ; + if(!ok) { + System.err.println("NcdumpW failed"); + } + return sw.toString(); } ////////////////////////////////////////////////// // Utility methods // Locate the test cases with given prefix - List + List locate(Object pattern) { - List results = new ArrayList(); - for(ConstraintTest ct : this.alltestcases) { + List results = new ArrayList(); + for(TestCase ct : this.alltestcases) { if(pattern instanceof String) { if(ct.title.equals(pattern.toString())) results.add(ct); @@ -305,22 +330,6 @@ public void testFilters() } return results; } - ////////////////////////////////////////////////// - // Stand alone - - static public void - main(String[] argv) - { - try { - new TestServlet().testServlet(); - } catch (Exception e) { - System.err.println("*** FAIL"); - e.printStackTrace(); - System.exit(1); - } - System.err.println("*** PASS"); - System.exit(0); - }// main } diff --git a/dap4/d4tests/src/test/java/dap4/test/TestH5Iosp.java b/dap4/d4tests/src/test/java/dap4/test/TestH5Iosp.java index f24051cd2e..9d3d0867da 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestH5Iosp.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestH5Iosp.java @@ -1,5 +1,6 @@ package dap4.test; +import dap4.core.util.DapUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -32,10 +33,10 @@ public class TestH5Iosp extends DapTestCommon ////////////////////////////////////////////////// // Constants - static protected String DATADIR = "src/test/data"; // relative to dap4 root - static protected String TESTDATADIR = DATADIR + "/resources/"; - static protected String BASELINEDIR = DATADIR + "/resources/TestIosp/baseline"; - static protected String TESTINPUTDIR = DATADIR + "/resources/testfiles"; + static protected String DATADIR = "d4tests/src/test/data"; // relative to dap4 root + static protected String TESTDATADIR = "/resources/"; + static protected String BASELINEDIR = "/resources/TestIosp/baseline"; + static protected String TESTINPUTDIR = "/resources/testfiles"; static protected final BigInteger MASK = new BigInteger("FFFFFFFFFFFFFFFF", 16); @@ -92,17 +93,17 @@ static protected enum Mode public void setup() throws Exception { this.testroot = getTestFilesDir(); - File f = new File(testroot + "/" + BASELINEDIR); - if(!f.exists()) f.mkdir(); this.datasetpath = this.testroot + "/" + DATADIR; - defineAllTestcases(this.testroot); + File f = new File(this.datasetpath + "/" + BASELINEDIR); + if(!f.exists()) f.mkdir(); + defineAllTestcases(this.datasetpath); chooseTestcases(); } protected String getTestFilesDir() { - return TESTINPUTDIR; + return getDAP4Root(); } ////////////////////////////////////////////////// @@ -112,7 +113,7 @@ public void setup() throws Exception chooseTestcases() { if(false) { - chosentests = locate("test_enum.nc"); + chosentests = locate("test_one_var.nc"); //chosentests.add(new H5IospTest("test_test.nc")); } else { for(H5IospTest tc : alltestcases) { diff --git a/dap4/d4tests/src/test/java/dap4/test/TestNc4Iosp.java b/dap4/d4tests/src/test/java/dap4/test/TestNc4Iosp.java index d8c28262b8..d6f15a2234 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestNc4Iosp.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestNc4Iosp.java @@ -1,12 +1,16 @@ package dap4.test; +import dap4.core.util.DapUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.nc2.dataset.NetcdfDataset; +import ucar.unidata.util.test.category.NotJenkins; +import java.io.File; import java.io.IOException; import java.io.StringWriter; import java.lang.invoke.MethodHandles; @@ -15,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +@Category(NotJenkins.class) public class TestNc4Iosp extends DapTestCommon { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -28,9 +33,10 @@ public class TestNc4Iosp extends DapTestCommon ////////////////////////////////////////////////// // Constants - static protected final String RESOURCEPATH = "/src/test/data/resources"; // wrt getTestInputFilesDIr - static protected final String TESTINPUTDIR = "/testfiles"; - static protected final String BASELINEDIR = "/TestIosp/baseline"; + static protected String DATADIR = "d4tests/src/test/data"; // relative to dap4 root + static protected String TESTDATADIR = "/resources/"; + static protected String BASELINEDIR = "/resources/TestIosp/baseline"; + static protected String TESTINPUTDIR = "/resources/testfiles"; static protected final BigInteger MASK = new BigInteger("FFFFFFFFFFFFFFFF", 16); @@ -39,16 +45,7 @@ public class TestNc4Iosp extends DapTestCommon static protected class Nc4IospTest { - static String inputroot = null; - static String baselineroot = null; - - static public void - setRoots(String input, String baseline) - { - inputroot = input; - baselineroot = baseline; - } - + static String root = null; String title; String dataset; String testinputpath; @@ -58,8 +55,10 @@ static protected class Nc4IospTest { this.title = dataset; this.dataset = dataset; - this.testinputpath = canonjoin(this.inputroot, dataset); - this.baselinepath = canonjoin(this.baselineroot, dataset) + ".nc4"; + this.testinputpath + = root + "/" + TESTINPUTDIR + "/" + dataset; + this.baselinepath + = root + "/" + BASELINEDIR + "/" + dataset + ".nc4"; } public String toString() @@ -81,30 +80,30 @@ static protected enum Mode // Test cases - protected List alltestcases = new ArrayList(); + protected List alltestcases = new ArrayList<>(); - protected List chosentests = new ArrayList(); + protected List chosentests = new ArrayList<>(); protected String datasetpath = null; - protected String root = null; + protected String testroot = null; ////////////////////////////////////////////////// @Before public void setup() throws Exception { - this.root = getResourceRoot(); - testSetup(); - Nc4IospTest.setRoots(canonjoin(getResourceRoot(), TESTINPUTDIR), - canonjoin(getResourceRoot(), BASELINEDIR)); - defineAllTestcases(); + this.testroot = getTestFilesDir(); + this.datasetpath = this.testroot + "/" + DATADIR; + File f = new File(this.datasetpath + "/" + BASELINEDIR); + if(!f.exists()) f.mkdir(); + defineAllTestcases(this.datasetpath); chooseTestcases(); } protected String getTestFilesDir() { - return ""; + return getDAP4Root(); } ////////////////////////////////////////////////// @@ -114,20 +113,19 @@ public void setup() throws Exception chooseTestcases() { if(false) { - chosentests = locate("test_struct_array.nc"); - prop_visual = true; - prop_debug = true; + chosentests = locate("test_one_var.nc"); //chosentests.add(new Nc4IospTest("test_test.nc")); } else { - prop_baseline = false; for(Nc4IospTest tc : alltestcases) { chosentests.add(tc); } } } - void defineAllTestcases() + // Depending on which libnetcdf4-wrapper is used, the enum names will differ + void defineAllTestcases(String root) { + Nc4IospTest.root = root; this.alltestcases.add(new Nc4IospTest("test_one_var.nc")); this.alltestcases.add(new Nc4IospTest("test_one_vararray.nc")); this.alltestcases.add(new Nc4IospTest("test_atomic_types.nc")); @@ -153,17 +151,21 @@ public void testNc4Iosp() throws Exception { for(Nc4IospTest testcase : chosentests) { - doOneTest(testcase); + if(!doOneTest(testcase)) { + Assert.assertTrue(false); + } } } ////////////////////////////////////////////////// // Primary test method - void + boolean doOneTest(Nc4IospTest testcase) throws Exception { - System.err.println("Testcase: " + testcase.testinputpath); + boolean pass = true; + + System.out.println("Testcase: " + testcase.testinputpath); NetcdfDataset ncfile = openDataset(testcase.testinputpath); @@ -181,8 +183,6 @@ public void testNc4Iosp() } String baselinefile = String.format("%s", testcase.baselinepath); - System.err.println("Testpath: " + testcase.testinputpath); - System.err.println("Baseline: " + baselinefile); if(prop_baseline) { if(mode == Mode.DMR || mode == Mode.BOTH) writefile(baselinefile + ".dmr", metadata); @@ -192,26 +192,19 @@ public void testNc4Iosp() String baselinecontent = null; if(mode == Mode.DMR || mode == Mode.BOTH) { // Read the baseline file(s) - System.err.println("DMR Comparison:"); - try { - baselinecontent = readfile(baselinefile + ".dmr"); - boolean pass = same(getTitle(), baselinecontent, metadata); - Assert.assertTrue("***Fail", pass); - } catch (IOException ioe) { - Assert.assertTrue("baselinefile" + ".dmr: " + ioe.getMessage(), false); - } + System.out.println("DMR Comparison:"); + baselinecontent = readfile(baselinefile + ".dmr"); + pass = pass && same(getTitle(), baselinecontent, metadata); + System.out.println(pass ? "Pass" : "Fail"); } if(mode == Mode.DATA || mode == Mode.BOTH) { - System.err.println("DATA Comparison:"); - try { - baselinecontent = readfile(baselinefile + ".dap"); - Assert.assertTrue("***Data Fail", same(getTitle(), baselinecontent, data)); - - } catch (IOException ioe) { - Assert.assertTrue("baselinefile" + ".dap: " + ioe.getMessage(), false); - } + System.out.println("DATA Comparison:"); + baselinecontent = readfile(baselinefile + ".dap"); + pass = pass && same(getTitle(), baselinecontent, data); + System.out.println(pass ? "Pass" : "Fail"); } } + return pass; } ////////////////////////////////////////////////// @@ -265,13 +258,11 @@ String ncdumpmetadata(NetcdfDataset ncfile, String datasetname) sw.close(); } catch (IOException e) { } - if(!ok) { System.err.println("NcdumpW failed"); System.exit(1); } - //return shortenFileName(sw.toString(), ncfile.getLocation()); - return sw.toString(); + return shortenFileName(sw.toString(), ncfile.getLocation()); } String ncdumpdata(NetcdfDataset ncfile, String datasetname) @@ -284,7 +275,6 @@ String ncdumpdata(NetcdfDataset ncfile, String datasetname) args.append(" -datasetname "); args.append(datasetname); } - // Dump the databuffer sw = new StringWriter(); ok = false; @@ -303,10 +293,8 @@ String ncdumpdata(NetcdfDataset ncfile, String datasetname) System.err.println("NcdumpW failed"); System.exit(1); } - //return shortenFileName(sw.toString(), ncfile.getLocation()); - return sw.toString(); + return shortenFileName(sw.toString(), ncfile.getLocation()); } - } diff --git a/dap4/d4tests/src/test/java/dap4/test/TestParserDMR.java b/dap4/d4tests/src/test/java/dap4/test/TestParserDMR.java index f2b3a69092..bd4260a4f9 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestParserDMR.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestParserDMR.java @@ -89,8 +89,9 @@ public TestParserDMR() chooseTestcases() { if(false) { - chosentests = locate("test_struct_nested.hdf5"); + chosentests = locate("test_ncml"); prop_visual = true; + prop_baseline = true; assert chosentests.size() > 0 : "No tests chosen"; } else { for(TestCase tc : alltestcases) { @@ -241,11 +242,11 @@ public void testParser() sw.close(); String testresult = sw.toString(); - // Read the baseline file - String baselinecontent; + // Read the baseline file, unless generating it + String baselinecontent = null; if(BACKCOMPARE) baselinecontent = document; - else + else if(!prop_baseline) baselinecontent = readfile(baseline); if(prop_visual) { visual("Baseline", baselinecontent); diff --git a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java index 58d609da88..1a73a28172 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java @@ -3,6 +3,7 @@ import dap4.core.util.DapUtil; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,27 +20,26 @@ /** * Test at the NetcdfDataset level; access .ser files on server. */ +@Ignore public class TestSerial extends DapTestCommon { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static protected final boolean DEBUG = false; - static protected final String TESTINPUTDIR = "/testfiles"; + static protected final String TESTINPUTDIR = "testfiles"; static protected final boolean NCDUMP = true; // Use NcDumpW instead of D4Print static protected final String EXTENSION = (NCDUMP ? "ncdump" : "dmp"); - static protected final String DAP4TAG = "#dap4"; - static protected final String[] EMPTY = new String[]{""}; ////////////////////////////////////////////////// // Constants static protected final String DATADIR = "src/test/data"; // relative to dap4 root - static protected final String TESTDATADIR = DATADIR + "/resources/TestCDMClient"; + static protected final String TESTDATADIR = "/TestCDMClient"; static protected final String BASELINEDIR = TESTDATADIR + "/baseline"; static protected final String alpha = "abcdefghijklmnopqrstuvwxyz" @@ -85,7 +85,7 @@ String makeurl(String ce) url.append(this.dataset); url.append("."); url.append("nc"); - url.append(DAP4TAG); + url.append(DAP4MODE); if(ce != null && ce.length() > 0) { url.append("?"); url.append(DapTestCommon.CONSTRAINTTAG); diff --git a/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java b/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java index cd5c1b08bb..600950f85b 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestServletConstraints.java @@ -127,7 +127,7 @@ String makequery() if(this.constraint != null) { String ce = this.constraint; // Escape it - ce = Escape.urlEncodeQuery(ce); + ce = Escape.urlEncode(ce); query = ce; } return query; diff --git a/tds/src/test/java/thredds/mock/web/MockTdsServletContext.java b/tds/src/test/java/thredds/mock/web/MockTdsServletContext.java index 13e6be9622..227e208138 100644 --- a/tds/src/test/java/thredds/mock/web/MockTdsServletContext.java +++ b/tds/src/test/java/thredds/mock/web/MockTdsServletContext.java @@ -83,6 +83,10 @@ protected String getResourceLocation(String path) { if (!path.startsWith("/")) { path = "/" + path; } - return "src/main/webapp" + path; + returnu "src/main/webapp" + path; } */ + + public String getVirtualServerName() { + return null; + } } diff --git a/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java b/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java index f24ff5ca94..87cd63184c 100644 --- a/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java +++ b/testUtil/src/main/java/ucar/unidata/util/test/UnitTestCommon.java @@ -442,6 +442,8 @@ public String getName() { StringBuilder buf = new StringBuilder(); File xx = new File(filename); + if(!xx.exists()) + throw new IOException("File does not exist: "+filename); FileReader file = new FileReader(filename); BufferedReader rdr = new BufferedReader(file); String line; From 065f9f690f5e13b06e6ece1baf81ceecb61f2754 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 21 Oct 2019 12:39:16 -0600 Subject: [PATCH 4/7] Fix test cases --- .../d4servlet/src/main/java/dap4/servlet/DapDSP.java | 12 ++++++++---- .../src/test/java/dap4/test/TestCDMClient.java | 3 ++- .../src/test/java/dap4/test/TestConstraints.java | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java index 236bcd1613..8f888d38f9 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java @@ -254,14 +254,18 @@ public static synchronized DSP open(DapRequest drq, String target, DapContext cx // See if this parses as a URL try { URI uri = new URI(target); + String scheme = uri.getScheme(); + if(scheme == null) + System.err.println("XXXX: "+target); + assert(scheme != null); // Windows drive letters cause URI to succeed, so special hack for that - if (uri.getScheme().length() == 1 && driveletters.indexOf(uri.getScheme().charAt(0)) >= 0) - throw new URISyntaxException("windows drive letter",target); + if(scheme.length() == 1 && driveletters.indexOf(scheme.charAt(0)) >= 0) + throw new URISyntaxException("windows drive letter", target); // If uri protocol is file, then extract the path - if(uri.getScheme().equals("file")) + if(scheme.equals("file")) path = uri.getPath(); else - dsp = open(uri,cxt); // open as general URI + dsp = open(uri, cxt); // open as general URI } catch (URISyntaxException use) { // assume it is a simple file path path = target; diff --git a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java index 605463f47a..7aa2441ea4 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestCDMClient.java @@ -229,7 +229,8 @@ else if(!f.canRead()) String url = "file://"+path; try { URL u = new URL(url); - System.err.printf("Testcase: add: %s path=%s%n",u.toString(),u.getPath()); + if(DEBUG) + System.err.printf("Testcase: add: %s path=%s%n",u.toString(),u.getPath()); } catch (MalformedURLException e) { System.err.println("Malformed file test case: " + url); } diff --git a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java index 091e351ff9..d77d47b4d5 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java @@ -153,8 +153,8 @@ public void setup() throws Exception void chooseTestcases() { - if(false) { - chosentests.add(locate1(4)); + if(true) { + chosentests.add(locate1(1)); prop_visual = true; } else { prop_baseline = false; From fd06c76f301f789fb2f52c0359d0c22d8aaaf7f6 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 21 Oct 2019 13:43:54 -0600 Subject: [PATCH 5/7] Fix flaw in identifying URL request --- .../src/main/java/dap4/servlet/DapDSP.java | 14 ++++++-------- .../src/test/java/dap4/test/TestConstraints.java | 2 +- .../src/test/java/dap4/test/TestSerial.java | 5 ++++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java index 8f888d38f9..3c53264cc4 100644 --- a/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java +++ b/dap4/d4servlet/src/main/java/dap4/servlet/DapDSP.java @@ -256,13 +256,11 @@ public static synchronized DSP open(DapRequest drq, String target, DapContext cx URI uri = new URI(target); String scheme = uri.getScheme(); if(scheme == null) - System.err.println("XXXX: "+target); - assert(scheme != null); - // Windows drive letters cause URI to succeed, so special hack for that - if(scheme.length() == 1 && driveletters.indexOf(scheme.charAt(0)) >= 0) - throw new URISyntaxException("windows drive letter", target); - // If uri protocol is file, then extract the path - if(scheme.equals("file")) + path = target; // apparently not a URL + else if(scheme.length() == 1 && driveletters.indexOf(scheme.charAt(0)) >= 0) + // Windows drive letters cause URI to succeed, so special hack for that + path = target; // really not a URL + else if(scheme.equals("file")) // If uri protocol is file, then extract the path path = uri.getPath(); else dsp = open(uri, cxt); // open as general URI @@ -271,7 +269,7 @@ public static synchronized DSP open(DapRequest drq, String target, DapContext cx path = target; } - String core = filepathcore(path); // remove any trailing .dmr|.dap + String core = filepathcore(path); // remove any trailing .dmr|.dap -- null ok if(dsp == null) { // See if this can open as a NetcdfFile|NetcdfDataset diff --git a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java index d77d47b4d5..c367164b86 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestConstraints.java @@ -153,7 +153,7 @@ public void setup() throws Exception void chooseTestcases() { - if(true) { + if(false) { chosentests.add(locate1(1)); prop_visual = true; } else { diff --git a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java index 1a73a28172..eeaadc58da 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestSerial.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestSerial.java @@ -5,11 +5,14 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.nc2.dataset.NetcdfDataset; import ucar.unidata.util.test.TestDir; import ucar.unidata.util.test.UnitTestCommon; +import ucar.unidata.util.test.category.NotJenkins; +import ucar.unidata.util.test.category.NotTravis; import java.io.IOException; import java.io.StringWriter; @@ -20,7 +23,6 @@ /** * Test at the NetcdfDataset level; access .ser files on server. */ -@Ignore public class TestSerial extends DapTestCommon { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -170,6 +172,7 @@ public void setup() throws Exception // Junit test method @Test + @Category({NotJenkins.class, NotTravis.class}) public void testSerial() throws Exception { From 7f873fa34553809b51bc308f06b9dbb6dddcd486 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 21 Oct 2019 15:06:02 -0600 Subject: [PATCH 6/7] missing ncml.dap file --- .../data/resources/TestServlet/baseline/test_ncml.ncml.dap | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dap diff --git a/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dap b/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dap new file mode 100644 index 0000000000..905b54a940 --- /dev/null +++ b/dap4/d4tests/src/test/data/resources/TestServlet/baseline/test_ncml.ncml.dap @@ -0,0 +1,4 @@ + 17 + 240 + 32700 + 111000 From 98b3a324bae4b8c0005d1bcf7b01e3cc8a431880 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 21 Oct 2019 15:12:10 -0600 Subject: [PATCH 7/7] Bad file: url --- dap4/d4tests/src/test/java/dap4/test/TestDSP.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java index 88429128e7..de6197bf9f 100644 --- a/dap4/d4tests/src/test/java/dap4/test/TestDSP.java +++ b/dap4/d4tests/src/test/java/dap4/test/TestDSP.java @@ -272,7 +272,7 @@ public void setup() throws Exception if(f.indexOf(x) >= 0) {excluded = true; break;} } if(!excluded) - add("file:/" + f); + add("file://" + f); } if(SHOWTESTCASES) { for(int i=0;i