Skip to content

[WIP] Added support for resolving vault secrets on-demand basis #1538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
VShingala wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

[WIP] Added support for resolving vault secrets on-demand basis #1538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 86 additions & 11 deletions lib/runner/extensions/event.command.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,20 +408,28 @@ module.exports = {
}
}.bind(this));

// Explicitly enable tracking for vault secrets here as this will
// not be sent to sandbox who otherwise takes care of mutation tracking
// This is important especially when dealing with nested requests, without this, the parent req might
// not have a pm.vault.<cmd> call and thus no mutations would bubble up and apply to the parent
// enable tracking for vault secrets when using old format (VariableScope)
if (vaultSecrets && vaultSecrets.enableTracking) {
vaultSecrets.enableTracking({ autoCompact: true });
}

this.host.on(EXECUTION_VAULT_BASE + executionId, async function (id, cmd, ...args) {
// check if using old format (VariableScope) or new format (with resolver)
var isOldFormat = vaultSecrets && typeof vaultSecrets.get === 'function',
vaultResolver = vaultSecrets && vaultSecrets.resolver,
vaultPrefix = (vaultSecrets && vaultSecrets.prefix) || 'vault:',
vaultCache = (vaultSecrets && vaultSecrets._cache) || {},
allowScriptAccess = isOldFormat ?
(vaultSecrets._ && vaultSecrets._.allowScriptAccess) :
(vaultSecrets && vaultSecrets.allowScriptAccess);

// check vault access on first call
if (hasVaultAccess === undefined) {
try {
// eslint-disable-next-line require-atomic-updates
hasVaultAccess = Boolean(this.state.nestedRequest?.hasVaultAccess ||
await vaultSecrets?._?.allowScriptAccess(rootItemId));
(typeof allowScriptAccess === 'function' &&
await allowScriptAccess(rootItemId)));
}
catch (_) {
// eslint-disable-next-line require-atomic-updates
Expand All @@ -433,8 +441,6 @@ module.exports = {
this.state.nestedRequest.hasVaultAccess = hasVaultAccess;
}

// Ensure error is string
// TODO identify why error objects are not being serialized correctly
const dispatch = (e, r) => { this.host.dispatch(EXECUTION_VAULT_BASE + executionId, id, e, r); };

if (!hasVaultAccess) {
Expand All @@ -445,7 +451,73 @@ module.exports = {
return dispatch(`Invalid vault command: ${cmd}`);
}

dispatch(null, vaultSecrets[cmd](...args));
// handle OLD format: use VariableScope methods directly
if (isOldFormat) {
return dispatch(null, vaultSecrets[cmd](...args));
}

// handle NEW format: use resolver for on-demand fetching
if (cmd === 'get') {
var key = args[0];

// check if value is in cache
if (Object.hasOwn(vaultCache, key)) {
return dispatch(null, vaultCache[key]);
}

// if no resolver, return undefined
if (typeof vaultResolver !== 'function') {
return dispatch(null, undefined);
}

// call resolver for on-demand fetch
try {
var resolverContext = {
keys: [key],
url: payload.context && payload.context.item &&
payload.context.item.request &&
payload.context.item.request.url &&
payload.context.item.request.url.toString(),
itemId: rootItemId,
executionId: executionId,
source: 'script'
};

vaultResolver(resolverContext, function (err, resolved) {
if (err || !Array.isArray(resolved) || resolved.length === 0) {
return dispatch(null, undefined);
}

// cache the resolved value
var variable = resolved[0];

if (variable && variable.key) {
var keyWithoutPrefix = variable.key.startsWith(vaultPrefix) ?
variable.key.substring(vaultPrefix.length) : variable.key;

vaultCache[keyWithoutPrefix] = variable.value;
vaultSecrets._cache = vaultCache;
}

dispatch(null, variable && variable.value);
});
}
catch (resolverErr) {
dispatch(null, undefined);
}
}
else if (cmd === 'set') {
// set value in cache
vaultCache[args[0]] = args[1];
vaultSecrets._cache = vaultCache;
dispatch(null, undefined);
}
else if (cmd === 'unset') {
// remove from cache
delete vaultCache[args[0]];
vaultSecrets._cache = vaultCache;
dispatch(null, undefined);
}
}.bind(this));

this.host.on(EXECUTION_REQUEST_EVENT_BASE + executionId,
Expand Down Expand Up @@ -596,11 +668,14 @@ module.exports = {
(result.collectionVariables = new sdk.VariableScope(result.collectionVariables));
result && result.request && (result.request = new sdk.Request(result.request));

// vault secrets are not sent to sandbox, thus using the scope from run context.
// vault secrets handling in result
if (vaultSecrets) {
// Prevent mutations from being carry-forwarded to subsequent events
vaultSecrets.disableTracking();
// disable tracking for old format to prevent mutations carry-forward
if (typeof vaultSecrets.disableTracking === 'function') {
vaultSecrets.disableTracking();
}

// pass vault secrets to result if access was granted
if (hasVaultAccess || this.state.nestedRequest?.hasVaultAccess) {
result.vaultSecrets = vaultSecrets;
}
Expand Down
45 changes: 25 additions & 20 deletions lib/runner/extensions/request.command.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
var _ = require('lodash'),
createItemContext = require('../create-item-context'),
{ resolveVariables } = require('../util');
util = require('../util');


module.exports = {
Expand All @@ -12,38 +12,43 @@ module.exports = {

process: {
request (payload, next) {
var abortOnError = _.has(payload, 'abortOnError') ? payload.abortOnError : this.options.abortOnError,
var self = this,
abortOnError = _.has(payload, 'abortOnError') ? payload.abortOnError : this.options.abortOnError,

// helper function to trigger `response` callback anc complete the command
// helper function to trigger `response` callback and complete the command
complete = function (err, nextPayload) {
// nextPayload will be empty for unhandled errors
// trigger `response` callback
// nextPayload.response will be empty for error flows
// the `item` argument is resolved and mutated here
nextPayload && this.triggers.response(err, nextPayload.coords, nextPayload.response,
nextPayload && self.triggers.response(err, nextPayload.coords, nextPayload.response,
nextPayload.request, nextPayload.item, nextPayload.cookies, nextPayload.history);

// the error is passed twice to allow control between aborting the error vs just
// bubbling it up
return next(err && abortOnError ? err : null, nextPayload, err);
}.bind(this),
},
context = createItemContext(payload);

// resolve variables in item and auth
resolveVariables(context, payload);

// add context for use, after resolution
payload.context = context;

// we do not queue `httprequest` instruction here,
// queueing will unblock the item command to prepare for the next `event` instruction
// at this moment request is not fulfilled, and we want to block it
this.immediate('httprequest', payload)
.done(function (nextPayload, err) {
// change signature to error first
complete(err, nextPayload);
})
.catch(complete);
// resolve variables in item and auth (supports async vault resolution)
util.resolveVariables(context, payload, function (err) {
if (err) {
return complete(err);
}

// add context for use, after resolution
payload.context = context;

// we do not queue `httprequest` instruction here,
// queueing will unblock the item command to prepare for the next `event` instruction
// at this moment request is not fulfilled, and we want to block it
self.immediate('httprequest', payload)
.done(function (nextPayload, httpErr) {
// change signature to error first
complete(httpErr, nextPayload);
})
.catch(complete);
});
}
}
};
13 changes: 9 additions & 4 deletions lib/runner/extensions/waterfall.command.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ var _ = require('lodash'),
{
getIterationData,
prepareVariablesScope,
processExecutionResult,
prepareVaultVariableScope
prepareVaultVariableScope,
processExecutionResult
} = require('../util');

/**
Expand All @@ -17,9 +17,14 @@ module.exports = {
init: function (done) {
var state = this.state;

// prepare the vault variable scope and other variables
// prepare variable scopes
prepareVariablesScope(state);
prepareVaultVariableScope(state.vaultSecrets);

// prepare vault scope for old format (with values)
// new format (with resolver) doesn't need this
if (state.vaultSecrets && state.vaultSecrets.values && !state.vaultSecrets.resolver) {
prepareVaultVariableScope(state.vaultSecrets);
}

// ensure that the items and iteration data set is in place
!_.isArray(state.items) && (state.items = []);
Expand Down
36 changes: 25 additions & 11 deletions lib/runner/nested-request.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,30 @@ function runNestedRequest ({ executionId, isExecutionSkipped, vaultSecrets, item
// Merge local variables from parent requests & scope + nestedRequest.options.variables
localVariables.values = [...localVariables.values, ...variableOverrides];

// Why clone? Each runner execution needs to track and mutate its vault variables separately and propagate
// it back up and further down. We don't want to accidentally reset mutations between executions by sharing
// this scope
// clone vault secrets for nested requests
// each runner execution needs its own scope to track mutations separately
let clonedVaultSecrets;

if (vaultSecrets) {
clonedVaultSecrets = new sdk.VariableScope({
values: vaultSecrets.values,
prefix: vaultSecrets.prefix
});
// check if using old format (VariableScope) or new format (with resolver)
if (typeof vaultSecrets.get === 'function') {
// old format: clone the VariableScope
clonedVaultSecrets = new sdk.VariableScope({
values: vaultSecrets.values,
prefix: vaultSecrets.prefix
});

clonedVaultSecrets._ = vaultSecrets._;
clonedVaultSecrets._ = vaultSecrets._;
}
else {
// new format: share cache with parent so resolved secrets propagate
clonedVaultSecrets = {
prefix: vaultSecrets.prefix,
resolver: vaultSecrets.resolver,
allowScriptAccess: vaultSecrets.allowScriptAccess,
_cache: vaultSecrets._cache || {}
};
}
}

new runner().run(runnableRefRequestCollection,
Expand Down Expand Up @@ -143,13 +155,13 @@ function runNestedRequest ({ executionId, isExecutionSkipped, vaultSecrets, item
}
run.start({
script (_err, _cursor, result) {
// This is to sync changes to pm.variables, pm.environment & pm.globals
// sync changes to pm.variables, pm.environment & pm.globals
// that happened inside the nested request's script
// back to parent request's scripts still currently executing.
// back to parent request's scripts still currently executing

// collectionVariables don't need to be synced between parent and nested

// All other global variables defined by syntax like 'a=1'
// all other global variables defined by syntax like 'a=1'
// are anyway synced as the sandbox's common scope is shared across runs
if (result) {
SYNCABLE_CONTEXT_VARIABLE_SCOPES.forEach(function (type) {
Expand All @@ -163,11 +175,13 @@ function runNestedRequest ({ executionId, isExecutionSkipped, vaultSecrets, item
];
});

// sync vault mutations for old format (VariableScope)
if (clonedVaultSecrets && clonedVaultSecrets.mutations) {
const mutations = new sdk.MutationTracker(clonedVaultSecrets.mutations);

mutations.applyOn(vaultSecrets);
}
// new format: cache is shared by reference, no explicit sync needed
}
},
request (err, _cursor, ...rest) {
Expand Down
21 changes: 14 additions & 7 deletions lib/runner/replay-controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
var _ = require('lodash'),
createItemContext = require('./create-item-context'),
{ resolveVariables } = require('./util'),
util = require('./util'),

// total number of replays allowed
MAX_REPLAY_COUNT = 3,
Expand Down Expand Up @@ -33,6 +33,8 @@ _.assign(ReplayController.prototype, /** @lends ReplayController.prototype */{
* @param {Function} failure this callback is invoked when replay controller decided not to send the request
*/
requestReplay (context, item, desiredPayload, success, failure) {
var self = this;

// max retries exceeded
if (this.count >= MAX_REPLAY_COUNT) {
return failure(new Error('runtime: maximum intermediate request limit exceeded'));
Expand All @@ -55,13 +57,18 @@ _.assign(ReplayController.prototype, /** @lends ReplayController.prototype */{
// create item context from the new item
payload.context = createItemContext(payload, context);

resolveVariables(payload.context, payload);
// resolve variables asynchronously (supports on-demand vault resolution)
util.resolveVariables(payload.context, payload, function (err) {
if (err) {
return success(err);
}

this.run.immediate('httprequest', payload)
.done(function (response) {
success(null, response);
})
.catch(success);
self.run.immediate('httprequest', payload)
.done(function (response) {
success(null, response);
})
.catch(success);
});
},

/**
Expand Down
Loading
Loading