Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Common/columnencoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,22 @@ ColumnEncoder::colVec ColumnEncoder::columnNamesEncoded()
return _columnEncoder ? _columnEncoder->_encodedNames : colVec();
}

ColumnEncoder::colMap ColumnEncoder::decodingMapSnapshot()
{
columnEncoder();
return decodingMap();
}

std::string ColumnEncoder::decodeAllWithMapping(const std::string & text, const ColumnEncoder::colMap & decodingMap)
{
ColumnEncoder::colVec encodedNames;
for(const auto & keyVal : decodingMap)
encodedNames.push_back(keyVal.first);

sortVectorBigToSmall(encodedNames);
return replaceAll(text, decodingMap, encodedNames);
}

void ColumnEncoder::_convertPreloadingDataOption(Json::Value & options, const std::string& optionName, colsPlusTypes& colTypes)
{
std::string optionKey = options[optionName].isMember("optionKey") ? options[optionName]["optionKey"].asString() : "";
Expand Down Expand Up @@ -827,6 +843,7 @@ void ColumnEncoder::_addTypeToColumnNamesInOptionsRecursively(Json::Value & opti

ColumnEncoder::colsPlusTypes ColumnEncoder::encodeColumnNamesinOptions(Json::Value & options, bool preloadingData)
{
columnEncoder();
colsPlusTypes getTheseCols;

_addTypeToColumnNamesInOptionsRecursively(options, preloadingData, getTheseCols);
Expand Down
4 changes: 3 additions & 1 deletion Common/columnencoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ private: ColumnEncoder() { invalidateAll(); }

static colVec columnNames();
static colVec columnNamesEncoded();
static colMap decodingMapSnapshot();
static std::string decodeAllWithMapping(const std::string & text, const colMap & decodingMap);

bool shouldEncode(const std::string & in);
bool shouldDecode(const std::string & in);
Expand All @@ -87,7 +89,7 @@ private: ColumnEncoder() { invalidateAll(); }
static std::string encodeAll(const std::string & text) { return replaceAll(text, encodingMap(), originalNames()); }

///Replace all occurences of encoded columnNames in a string by their decoded versions, regardless of word boundaries or parentheses.
static std::string decodeAll(const std::string & text) { return replaceAll(text, decodingMap(), encodedNames()); }
static std::string decodeAll(const std::string & text) { columnEncoder(); return replaceAll(text, decodingMap(), encodedNames()); }

///Replace all occurences of columnNames in a string by their encoded versions in all json-names and string-values, regardless of word boundaries or parentheses.
static void encodeJson(Json::Value & json, bool replaceNames = false, bool replaceStrict = false);
Expand Down
14 changes: 13 additions & 1 deletion Common/log.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,19 @@

std::ofstream Log::_logFile;// = bofstream();

std::ostream* Log::_nullStream = &std::cout;
namespace
{
class NullLogBuffer : public std::streambuf
{
protected:
int overflow(int c) override { return traits_type::not_eof(c); }
};

NullLogBuffer nullLogBuffer;
std::ostream nullLogStream(&nullLogBuffer);
}

std::ostream* Log::_nullStream = &nullLogStream;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe this is a good idea but I will need to know why this was done?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is needed so logType::null actually discards output in the static jaspSyntax bridge. Before this change _nullStream pointed at std::cout, so syntaxBridgeSetVerbose(false) could still let native bridge logging reach stdout before Log::init() installed an application-owned null stream. The local streambuf is a true sink; verbose mode still switches the bridge back to stdout. I added a short code comment for that rationale.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could just call the following in the syntaxbridge constructor:

Log::logFileNameBase = (AppDirs::logDir() + "JASP "  + getSortableTimestamp()).toStdString();
Log::init(&nullstream);
Log::setLogFileName(Log::logFileNameBase + " Desktop.log");
Log::setLoggingToFile(_preferences->logToFile());

Like MainWindow::initLog ?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is exactly the issue this was trying to fix: when the static bridge is used from R/jaspSyntax, MainWindow::initLog() is never called, so logType::null still pointed at std::cout and JASP/native messages leaked into R sessions even with verbose disabled.

I agree this is bridge initialization concern rather than part of the decoder API. Before I patch it: do you want the SyntaxInterface bridge to only install a local null stream via Log::init(&nullStream) and then toggle Log::setDefaultDestination() / Log::setWhere() from syntaxBridgeSetVerbose(), or do you want the bridge to do the fuller MainWindow::initLog()-style log-file setup as well?

My preference would be the small bridge-local null-stream initialization, without Desktop preferences/log-file setup, because the R/headless bridge should not depend on PreferencesModel or Desktop-owned logging policy.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could use a nullstream like the one in MainWindow::initLog. And then Log::init(&nullstream). Much simpler than the code in this PR. And ofc in the syntaxbridge.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in c21e37f. I reverted the Common/log.cpp change entirely, so the global logging defaults outside SyntaxInterface are unchanged.

SyntaxInterface now owns a local null stream and calls Log::init(&nullstream) from configureBridgeLogging() before bridge status/error paths can log. syntaxBridgeSetVerbose() only toggles between cout and null; no Desktop preference/file logging is needed for this standalone bridge path. I also smoke-tested the jaspSyntax status-only error path with verbose = FALSE; it returns the structured error with zero captured native output.


std::string Log::logFileNameBase = "";

Expand Down
163 changes: 151 additions & 12 deletions SyntaxInterface/syntaxbridge.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@
#include "modules/dynamicmodule.h"
#include "archivereader.h"
#include "databaseinterface.h"
#include "columnencoder.h"

#include <string>
#include <vector>
#include <stdexcept>

#include <QtPlugin>
#ifdef USE_QT_STATIC_LIBS
Expand Down Expand Up @@ -152,6 +154,98 @@ static const char* statusError(Json::Value status, const std::string & error)
return statusResult(status);
}

static Json::Value columnDecoderSnapshotJson()
{
Json::Value snapshot(Json::objectValue);
Json::Value columns(Json::arrayValue);
const ColumnEncoder::colMap decodingMap = ColumnEncoder::decodingMapSnapshot();

for(const auto & keyVal : decodingMap)
{
Json::Value column(Json::objectValue);
column["encoded"] = keyVal.first;
column["decoded"] = keyVal.second;
columns.append(column);
}

snapshot["version"] = 1;
snapshot["columns"] = columns;
return snapshot;
}

struct ColumnDecoderSnapshot
{
ColumnEncoder::colMap decodingMap;
bool supplied = false;
};

static ColumnDecoderSnapshot columnDecoderMapFromSnapshot(const Json::Value & snapshot)
{
ColumnDecoderSnapshot snapshotState;
snapshotState.supplied = true;
const Json::Value & columns = snapshot["columns"];
if(!columns.isArray())
return snapshotState;

for(const Json::Value & column : columns)
if(column.isObject() && column["encoded"].isString() && column["decoded"].isString())
snapshotState.decodingMap[column["encoded"].asString()] = column["decoded"].asString();

return snapshotState;
}

static ColumnDecoderSnapshot columnDecoderMapFromSnapshotString(const char * snapshotJson)
{
if(!snapshotJson || std::string(snapshotJson).empty())
return ColumnDecoderSnapshot();

Json::Value snapshot;
Json::Reader reader;
if(!reader.parse(snapshotJson, snapshot))
throw std::runtime_error("Could not parse column decoder snapshot JSON.");

return columnDecoderMapFromSnapshot(snapshot);
}

static Json::Value parseStringArrayJson(const char * valuesJson)
{
if(!valuesJson)
throw std::runtime_error("Cannot decode column text from a null JSON payload.");

Json::Value values;
Json::Reader reader;
if(!reader.parse(valuesJson, values))
throw std::runtime_error("Could not parse column text JSON payload.");
if(!values.isArray())
throw std::runtime_error("Column text JSON payload must be an array.");

return values;
}

static Json::Value decodeColumnTextJson(const Json::Value & values, const ColumnDecoderSnapshot & snapshot)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole function looks like it should just call setCurrentColumnNames first with the stuff from the snapshot. And than ColumnEncoder::decodeJson. Not really sure why we need to have this whole function.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, this should be simplified. I can change the native path to parse the incoming JSON, temporarily install the captured context into ColumnEncoder, call the standard ColumnEncoder::decodeJson(), and restore the previous state before returning.

One detail to confirm: should this bridge call decode arbitrary JSON with replaceNames = true, or keep the current R contract as a character-vector JSON payload and still use decodeJson() internally? The former is more directly the standard ColumnEncoder API; the latter keeps the R API smaller and avoids exposing more surface than jaspSyntax currently needs.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it is the standard API for ColumnEncoder it is already accessible from R via jaspBase right? So that should make "The R API smaller" than the latter? Or am I missing something?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are not missing much; my phrasing about "R API smaller" was imprecise.

jaspBase::decodeColNames() / decodeName() are module-facing convenience helpers. They eventually resolve the decoder functions available in the current JASP runtime, but they are not a standalone way for jaspSyntax to install a saved Desktop ColumnEncoder context after the live dataset has changed. The dependency direction should also stay jaspBase -> jaspSyntax/native bridge for replay, not jaspSyntax -> jaspBase.

So I think the native bridge still needs one minimal context-aware decode entry point. But I agree it should be generic and standard-ColumnEncoder-shaped: context JSON + payload JSON in, install context, call ColumnEncoder::decodeJson(), restore, return decoded JSON. Then the exported R helper can remain a thin convenience wrapper for character vectors if that is all current callers need.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok well im curious to see it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in c21e37f. The bridge-specific manual JSON decoding loop is gone.

decodeColumnJson() now parses the payload, installs the snapshot context with ScopedColumnEncoderContext, calls the standard ColumnEncoder::decodeJson(payload, replaceNames), and returns the decoded payload. The payload is no longer restricted to a string array, so the normal ColumnEncoder traversal remains the single implementation for decoded JSON objects.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The standard decoder implementation is still the one we use, but replay needs to decode against the context captured from the source file/analysis, not whatever dataset context happens to be live when R materializes the result.

That is why I kept this as a single SyntaxInterface bridge operation instead of routing state management through jaspBase: the only cross-ABI API is decode(payload, context), and native code handles install/decode/restore internally. jaspSyntax remains a thin caller and jaspBase does not need to know about encoder mappings.

{
Json::Value decodedValues(Json::arrayValue);

for(const Json::Value & value : values)
{
if(value.isNull())
{
decodedValues.append(Json::Value());
}
else if(value.isString())
{
const std::string text = value.asString();
decodedValues.append(snapshot.supplied ? ColumnEncoder::decodeAllWithMapping(text, snapshot.decodingMap) : ColumnEncoder::decodeAll(text));
}
else
{
throw std::runtime_error("Column text JSON payload must contain only strings or null values.");
}
}

return decodedValues;
}

static Json::Value analysisOptionsStatus(const char * filePath, int analysisNr)
{
Json::Value status = statusBase("syntaxBridgeAnalysisOptionsFromJaspFile");
Expand Down Expand Up @@ -459,11 +553,22 @@ const char* STDCALL syntaxBridgeLoadDataSetFromJaspFileStatus(const char * fileP

const char* STDCALL syntaxBridgeLoadQmlAndParseOptions(const char* moduleName, const char* analysisName, const char* qmlFile, const char* options, const char* version, bool preloadData)
{
if (!init())
{
Log::log() << "Error during initialization" << std::endl;
Json::Value status;
Json::Reader reader;
if (!reader.parse(syntaxBridgeLoadQmlAndParseOptionsStatus(moduleName, analysisName, qmlFile, options, version, preloadData), status))
return "";
}
if (!status["ok"].asBool())
return "";

static std::string result;
result = status["options"].toStyledString();
return result.c_str();
}

const char* STDCALL syntaxBridgeLoadQmlAndParseOptionsStatus(const char* moduleName, const char* analysisName, const char* qmlFile, const char* options, const char* version, bool preloadData)
{
if (!init())
return statusError(statusBase("syntaxBridgeLoadQmlAndParseOptions"), "Error during initialization.");

std::string qmlFileStr = qmlFile,
versionStr = version,
Expand All @@ -475,17 +580,15 @@ const char* STDCALL syntaxBridgeLoadQmlAndParseOptions(const char* moduleName, c

if (!form)
{
Log::log() << "Cannot create QML Form " << qmlFileStr << std::endl;
return "";
return statusError(statusBase("syntaxBridgeLoadQmlAndParseOptions"), "Cannot create QML Form " + qmlFileStr);
}

Json::Value parsedOptions;
std::string errorMsg;

if (!form->parseOptions(options, parsedOptions, errorMsg))
{
Log::log() << "Error when parsing options: " << errorMsg << std::endl;
return "";
return statusError(statusBase("syntaxBridgeLoadQmlAndParseOptions"), "Error when parsing options: " + errorMsg);
}

gl_extraEncodings->setCurrentNamesFromOptionsMeta(parsedOptions);
Expand All @@ -494,10 +597,10 @@ const char* STDCALL syntaxBridgeLoadQmlAndParseOptions(const char* moduleName, c

rbridge_setWantedCols(analysisColsTypes);

static std::string result;
result = parsedOptions.toStyledString();

return result.c_str();
Json::Value status = statusBase("syntaxBridgeLoadQmlAndParseOptions");
status["ok"] = true;
status["options"] = parsedOptions;
return statusResult(status);
}

const char* STDCALL syntaxBridgeAnalysisOptionsFromJaspFile(const char * filePath, int analysisNr)
Expand Down Expand Up @@ -655,6 +758,42 @@ const char* STDCALL syntaxBridgeGetVariableNames()
return result.c_str();
}

void STDCALL syntaxBridgeSetVerbose(bool verbose)
{
gl_verbose = verbose;
Log::setDefaultDestination(verbose ? logType::cout : logType::null);
Log::setWhere(verbose ? logType::cout : logType::null);
}

const char* STDCALL syntaxBridgeColumnDecoderSnapshot()
{
static std::string result;

result = columnDecoderSnapshotJson().toStyledString();
return result.c_str();
}

const char* STDCALL syntaxBridgeDecodeColumnText(const char* valuesJson, const char* decoderSnapshotJson)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why isnt the decoder snapshot json not simply set from R? And the standard ColumnEncoder functions used? WHy all these extra functions?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I understand the intended simplification: R should not carry mappings or use bridge-specific replacement logic; it should pass the captured native context back, native code should install that context into ColumnEncoder, and then call standard ColumnEncoder decode functions.

The point I want to clarify before patching is whether you prefer a persistent R-facing set-current-ColumnEncoder-context API, or a scoped decode call that installs the snapshot, calls ColumnEncoder::decodeJson(), and restores the previous native state before returning.

I prefer the scoped form because result replay often happens after the live dataset has changed, and persistent mutation from R is easier to misuse. But it would still remove the custom decode machinery and use the normal ColumnEncoder path. Is that aligned with what you intended?

Related detail: should the same context continue to carry the extra QML-option encoder state (JaspExtraOptions_...) and restore that alongside the dataset column encoder?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes indeed, R should not carry mappings or bridge specific logic.

A scoped decode would require a bunch of extra interfacing functions right?
Why not simply make sure to set the right context from R? or is that where the scope would be?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No extra public bridge functions should be needed. My intended scoped version is a single native decode call: R passes the captured context JSON and payload JSON, SyntaxInterface installs that context into ColumnEncoder for that call, calls ColumnEncoder::decodeJson(), and restores the previous context before returning.

So the scope is inside the native decode call, not an R-side sequence of set/decode/reset calls. That keeps the R API from leaving global native decoder state changed after replay, but still uses the standard ColumnEncoder path. I will implement it that way unless you prefer the persistent set-current-context API.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SyntaxInterface installs that context into ColumnEncoder for that call, calls ColumnEncoder::decodeJson(), and restores the previous context before returning.

How?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mechanically:

  1. Parse the context JSON into the same ColumnEncoder::colTypeMap that came from DataSet::getColumnTypesMap() plus the extra-options encoder map.
  2. Save the currently active maps:
    • ColumnEncoder::columnEncoder()->currentNames()
    • gl_extraEncodings->currentNames()
  3. Install the captured maps:
    • ColumnEncoder::setCurrentColumnNames(contextColumns)
    • gl_extraEncodings->setCurrentNames(contextExtra)
  4. Parse the payload JSON and call the standard native decoder:
    • ColumnEncoder::decodeJson(payload, replaceNames)
  5. Restore the previously saved maps in an RAII guard/destructor before returning.

So the only custom code is context JSON <-> colTypeMap plus the scoped restore guard. The actual replacement would be ColumnEncoder::decodeJson(), not a separate bridge-specific decoding loop.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok sounds good.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Implemented in c21e37f with the scope inside the single native bridge call.

R/jaspSyntax still passes only the encoded payload plus the opaque encoder context. SyntaxInterface parses that context, ScopedColumnEncoderContext saves the current ColumnEncoder state, installs the snapshot names/types on ColumnEncoder and the extra encoder, calls the standard ColumnEncoder::decodeJson(), then restores the previous native state before returning. So R does not carry mappings and there are no separate set/unset calls exposed across the ABI.

{
static std::string result;

try
{
Json::Value values = parseStringArrayJson(valuesJson);
ColumnDecoderSnapshot snapshot = columnDecoderMapFromSnapshotString(decoderSnapshotJson);
result = decodeColumnTextJson(values, snapshot).toStyledString();
return result.c_str();
}
catch(const std::exception & exception)
{
return statusError(statusBase("syntaxBridgeDecodeColumnText"), exception.what());
}
catch(...)
{
return statusError(statusBase("syntaxBridgeDecodeColumnText"), "Unknown error while decoding column text.");
}
}

} // extern "C"


Expand Down
4 changes: 4 additions & 0 deletions SyntaxInterface/syntaxbridge_interface.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,16 @@ SYNTAX_INTERFACE void STDCALL syntaxBridgeLoadDataSet(const SyntaxBridgeDataS
SYNTAX_INTERFACE void STDCALL syntaxBridgeLoadDataSetFromJaspFile(const char * filePath, bool dbInMemory);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeLoadDataSetFromJaspFileStatus(const char * filePath, bool dbInMemory);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeLoadQmlAndParseOptions(const char * moduleName, const char* analysisName, const char* qmlFile, const char* options, const char* version, bool preloadData);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeLoadQmlAndParseOptionsStatus(const char * moduleName, const char* analysisName, const char* qmlFile, const char* options, const char* version, bool preloadData);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeAnalysisOptionsFromJaspFile(const char * filePath, int analysisNr);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeAnalysisOptionsFromJaspFileStatus(const char * filePath, int analysisNr);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeGenerateModuleWrappers(const char* name);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeGenerateAnalysisWrapper(const char* modulePath, const char* analysisName);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeParseDescription(const char* modulePath);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeGetVariableNames();
SYNTAX_INTERFACE void STDCALL syntaxBridgeSetVerbose(bool verbose);
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeColumnDecoderSnapshot();
SYNTAX_INTERFACE const char* STDCALL syntaxBridgeDecodeColumnText(const char* valuesJson, const char* decoderSnapshotJson);

} // extern "C"

Expand Down
Loading