Skip to content

Commit fee74d3

Browse files
committed
Updated protobuf design
1 parent a5b0a13 commit fee74d3

19 files changed

Lines changed: 1524 additions & 484 deletions

File tree

‎eng/pack/scripts/rust_deps.sh‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@
88
# compiled binary that embeds CPython via PyO3. This script:
99
# 1. compiles the release binary against the target interpreter,
1010
# 2. installs the v2 + v1 runtimes + azure-functions into $DEPS,
11-
# 3. materializes the bridge's private, gRPC-free protobuf stubs,
12-
# 4. overlays native_invocation.py / executor.py (the native + logging path),
13-
# 5. stages the binary, the bridge, and a noop.txt placeholder.
11+
# 3. overlays native_invocation.py / executor.py (the native + logging path),
12+
# 4. stages the binary and the bridge.
1413
#
1514
# The resulting $BUILD_SOURCESDIRECTORY/deps tree mirrors the runtime worker
1615
# directory produced by docker/Dockerfile and is copied into the NuGet under
@@ -49,20 +48,10 @@ export LD_LIBRARY_PATH="${PY_LIBDIR}:${LD_LIBRARY_PATH:-}"
4948
python -m pip install ./runtimes/v2 ./runtimes/v1 azure-functions \
5049
--no-compile --target "$DEPS"
5150

52-
# --- Version-correct protobuf message stubs -------------------------------
53-
# gen_protos.py copies proxy_worker/protos *_pb2.py stubs (version-matched to
54-
# the protobuf runtime) into the bridge's private `protos` package. Build them
55-
# first via the same invoke task the Python pack uses.
56-
python -m pip install invoke
57-
( cd workers && python -m pip install . && cd tests && python -m invoke -c test_setup build-protos )
58-
5951
# --- Stage the runtime worker-directory layout ----------------------------
6052
cp "$WORKER/target/release/rust_worker" "$DEPS/rust_worker"
6153
rm -rf "$DEPS/bridge"
6254
cp -r "$WORKER/bridge" "$DEPS/bridge"
63-
python "$WORKER/gen_protos.py" \
64-
--source "$BUILD_SOURCESDIRECTORY/workers/proxy_worker/protos" \
65-
--out "$DEPS/bridge/protos"
6655

6756
# Native (protobuf-free) invocation entry + executor overlay (invocation_id
6857
# correlation for user logs) for BOTH runtimes. Additive: only imports stable
@@ -76,9 +65,6 @@ cp runtimes/v1/azure_functions_runtime_v1/native_invocation.py \
7665
cp runtimes/v1/azure_functions_runtime_v1/utils/executor.py \
7766
"$DEPS/azure_functions_runtime_v1/utils/executor.py"
7867

79-
# Ignored positional so the Host has a valid defaultWorkerPath.
80-
touch "$DEPS/noop.txt"
81-
8268
cp workers/.artifactignore "$DEPS" 2>/dev/null || true
8369

8470
echo "Rust worker artifact staged under: $DEPS"

‎eng/pack/templates/rust_env_gen.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,8 @@ steps:
6565
# path.
6666
$expected = @(
6767
"rust_worker",
68-
"noop.txt",
6968
"bridge/bridge.py",
70-
"bridge/protos/FunctionRpc_pb2.py",
69+
"bridge/protos_adapter.py",
7170
"azure_functions_runtime",
7271
"azure_functions_runtime/native_invocation.py",
7372
"azure_functions_runtime/utils/executor.py",

‎eng/templates/shared/rust-build-steps.yml‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,21 +41,12 @@ steps:
4141
- bash: |
4242
set -euo pipefail
4343
python -m pip install --upgrade pip
44-
# Bridge + v2 runtime deps for the Python-only selftest and flake8.
45-
python -m pip install --only-binary=:all: 'grpcio>=1.75.1' 'protobuf>=5.29,<7'
44+
# v2 runtime (for flake8 import resolution) + flake8. The bridge is
45+
# protobuf-free (Rust/prost owns the wire), so no grpcio/protobuf here.
4646
python -m pip install -e runtimes/v2
4747
python -m pip install flake8
4848
displayName: 'Install Python dependencies'
4949
50-
- bash: |
51-
set -euo pipefail
52-
# Materialize the bridge's private, gRPC-free protobuf message stubs from
53-
# the version-correct proxy_worker stubs (matches the Docker build step).
54-
python workers/rust_worker/gen_protos.py \
55-
--source workers/proxy_worker/protos \
56-
--out workers/rust_worker/bridge/protos
57-
displayName: 'Generate bridge protobuf stubs'
58-
5950
- bash: |
6051
set -euo pipefail
6152
flake8 bridge

‎runtimes/v1/azure_functions_runtime_v1/native_invocation.py‎

Lines changed: 74 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -69,31 +69,81 @@ def __init__(self, invocation_id):
6969
self.retry_context = _RetryCtx()
7070

7171

72+
# --- Shared Datum-currency helpers -------------------------------------------
73+
#
74+
# The sync and async native paths differ ONLY in how the handler is executed.
75+
# Decoding inputs, preparing the invocation context, validating the return, and
76+
# encoding outputs are identical and protobuf-free, so they are factored out
77+
# here and reused by both entries below. (v1 has no deferred bindings, so
78+
# ``get_binding`` takes only the binding name.)
79+
80+
81+
def _decode_inputs(fi, inputs, trigger_metadata):
82+
"""Rust-provided ``(name, Datum)`` inputs -> decoded handler args dict."""
83+
metadata = trigger_metadata or {}
84+
args = {}
85+
for name, datum in inputs:
86+
pb_type_info = fi.input_types[name]
87+
tm = metadata if is_trigger_binding(pb_type_info.binding_name) else {}
88+
binding_obj = get_binding(pb_type_info.binding_name)
89+
args[name] = binding_obj.decode(datum, trigger_metadata=tm)
90+
return args
91+
92+
93+
def _prepare_context(fi, invocation_id, args):
94+
"""Build the invocation context and scaffold ``Out`` params into *args*.
95+
96+
Does NOT stamp ``thread_local_storage.invocation_id`` -- the sync path lets
97+
``run_sync_func`` do that on the executing thread, while the async path sets
98+
it explicitly on the caller's context (see below)."""
99+
fi_context = get_context(_InvocShim(invocation_id), fi.name, fi.directory)
100+
if fi.requires_context:
101+
args['context'] = fi_context
102+
if fi.output_types:
103+
for name in fi.output_types:
104+
args[name] = Out()
105+
return fi_context
106+
107+
108+
def _check_return(fi, call_result):
109+
if call_result is not None and not fi.has_return:
110+
raise RuntimeError(
111+
'function %s without a $return binding returned a non-None value'
112+
% repr(fi.name))
113+
114+
115+
def _collect_output(fi, args):
116+
"""Decoded ``Out`` params -> ``[(name, Datum)]`` for Rust to prost-encode."""
117+
output_data = []
118+
if fi.output_types:
119+
for out_name, out_type_info in fi.output_types.items():
120+
val = args[out_name].get()
121+
if val is None:
122+
continue
123+
datum = get_datum(out_type_info.binding_name, val,
124+
out_type_info.pytype)
125+
output_data.append((out_name, datum))
126+
return output_data
127+
128+
129+
def _encode_return(fi, call_result):
130+
if fi.return_type is not None:
131+
return get_datum(fi.return_type.binding_name, call_result,
132+
fi.return_type.pytype)
133+
return None
134+
135+
72136
async def invocation_request_native(invocation_id, function_id, inputs,
73137
trigger_metadata):
74138
threadpool = get_threadpool_executor()
75139
fi = _functions.get_function(function_id)
76140
if fi is None:
77141
return (False, None, [], "function %s not loaded" % function_id)
78142

79-
metadata = trigger_metadata or {}
80-
args = {}
81-
82143
try:
83-
for name, datum in inputs:
84-
pb_type_info = fi.input_types[name]
85-
tm = metadata if is_trigger_binding(pb_type_info.binding_name) else {}
86-
binding_obj = get_binding(pb_type_info.binding_name)
87-
args[name] = binding_obj.decode(datum, trigger_metadata=tm)
88-
89-
fi_context = get_context(_InvocShim(invocation_id), fi.name, fi.directory)
144+
args = _decode_inputs(fi, inputs, trigger_metadata)
145+
fi_context = _prepare_context(fi, invocation_id, args)
90146
fi_context.thread_local_storage.invocation_id = invocation_id
91-
if fi.requires_context:
92-
args['context'] = fi_context
93-
94-
if fi.output_types:
95-
for name in fi.output_types:
96-
args[name] = Out()
97147

98148
if fi.is_async:
99149
if otel_manager.get_azure_monitor_available():
@@ -115,25 +165,9 @@ async def invocation_request_native(invocation_id, function_id, inputs,
115165
threadpool, run_sync_func,
116166
invocation_id, fi_context, fi.func, args)
117167

118-
if call_result is not None and not fi.has_return:
119-
raise RuntimeError(
120-
'function %s without a $return binding returned a non-None value'
121-
% repr(fi.name))
122-
123-
output_data = []
124-
if fi.output_types:
125-
for out_name, out_type_info in fi.output_types.items():
126-
val = args[out_name].get()
127-
if val is None:
128-
continue
129-
datum = get_datum(out_type_info.binding_name, val,
130-
out_type_info.pytype)
131-
output_data.append((out_name, datum))
132-
133-
return_datum = None
134-
if fi.return_type is not None:
135-
return_datum = get_datum(fi.return_type.binding_name, call_result,
136-
fi.return_type.pytype)
168+
_check_return(fi, call_result)
169+
output_data = _collect_output(fi, args)
170+
return_datum = _encode_return(fi, call_result)
137171

138172
sys.stdout.flush()
139173
return (True, return_datum, output_data, None)
@@ -162,42 +196,14 @@ def run_invocation_sync(invocation_id, function_id, inputs, trigger_metadata):
162196
return (False, None, None, [], None)
163197

164198
try:
165-
metadata = trigger_metadata or {}
166-
args = {}
167-
for name, datum in inputs:
168-
pb_type_info = fi.input_types[name]
169-
tm = metadata if is_trigger_binding(pb_type_info.binding_name) else {}
170-
binding_obj = get_binding(pb_type_info.binding_name)
171-
args[name] = binding_obj.decode(datum, trigger_metadata=tm)
172-
173-
fi_context = get_context(_InvocShim(invocation_id), fi.name, fi.directory)
174-
if fi.requires_context:
175-
args['context'] = fi_context
176-
if fi.output_types:
177-
for name in fi.output_types:
178-
args[name] = Out()
199+
args = _decode_inputs(fi, inputs, trigger_metadata)
200+
fi_context = _prepare_context(fi, invocation_id, args)
179201

180202
call_result = run_sync_func(invocation_id, fi_context, fi.func, args)
181203

182-
if call_result is not None and not fi.has_return:
183-
raise RuntimeError(
184-
'function %s without a $return binding returned a non-None value'
185-
% repr(fi.name))
186-
187-
output_data = []
188-
if fi.output_types:
189-
for out_name, out_type_info in fi.output_types.items():
190-
val = args[out_name].get()
191-
if val is None:
192-
continue
193-
datum = get_datum(out_type_info.binding_name, val,
194-
out_type_info.pytype)
195-
output_data.append((out_name, datum))
196-
197-
return_datum = None
198-
if fi.return_type is not None:
199-
return_datum = get_datum(fi.return_type.binding_name, call_result,
200-
fi.return_type.pytype)
204+
_check_return(fi, call_result)
205+
output_data = _collect_output(fi, args)
206+
return_datum = _encode_return(fi, call_result)
201207

202208
return (True, True, return_datum, output_data, None)
203209

0 commit comments

Comments
 (0)