Skip to content

Commit 5d8c819

Browse files
waldekmastykarzCopilotgarrytrinder
authored
Restore system proxy after a crashed instance (#1756)
* Restore system proxy after a crashed instance When a detached instance running as system proxy is terminated uncleanly, the OS proxy was left pointing at a dead port and 'devproxy stop --force' could not recover it: StateManager prunes state files for dead PIDs before the stop command reads them, so no instance was found and the system proxy was never disabled. Reconcile orphaned system-proxy registrations independently of finding a live instance: - Add StateManager.GetOrphanedSystemProxyStatesAsync (reads records for dead PIDs with asSystemProxy=true without pruning them first). - Add cross-platform, idempotent SystemProxyManager.Disable (Windows WinINET via ProxyServer.DisableAllSystemProxies, macOS toggle-proxy.sh off) and ReconcileOrphanedSystemProxiesAsync, guarded so the OS proxy is only disabled when no live instance still owns it. - devproxy stop / stop --pid now restore the system proxy for crashed instances; self-heal also runs at startup (detached launcher and ProxyEngine) so a stale registration doesn't linger across runs. Refs #1731. PID-reuse hardening tracked in #1755. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0b22cd4a-15a3-4a8c-8248-fe776339aa45 * Address PR review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f2633a32-e727-4e47-93b9-76d9aeae01d6 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Garry Trinder <garry@trinder365.co.uk>
1 parent efae783 commit 5d8c819

5 files changed

Lines changed: 294 additions & 68 deletions

File tree

‎DevProxy/Commands/StopCommand.cs‎

Lines changed: 60 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using DevProxy.Proxy;
56
using DevProxy.State;
67
using System.CommandLine;
78
using System.Diagnostics;
@@ -34,18 +35,16 @@ private async Task<int> RunAsync(ParseResult parseResult, CancellationToken canc
3435

3536
if (pid is not null)
3637
{
37-
var state = await StateManager.LoadStateByPidAsync(pid.Value, cancellationToken);
38-
if (state is null)
39-
{
40-
Console.WriteLine($"No running Dev Proxy instance with PID {pid.Value}.");
41-
return 1;
42-
}
43-
44-
return await StopInstanceAsync(state, force, cancellationToken);
38+
return await StopByPidAsync(pid.Value, force, cancellationToken);
4539
}
4640

41+
// Reconcile any system-proxy registrations left behind by crashed
42+
// instances first — this must run before LoadAllStatesAsync, which prunes
43+
// (deletes) the stale state files it depends on.
44+
var reconciliation = await SystemProxyManager.ReconcileOrphanedSystemProxiesAsync(cancellationToken);
45+
4746
var states = await StateManager.LoadAllStatesAsync(cancellationToken);
48-
if (states.Count == 0)
47+
if (states.Count == 0 && reconciliation.Orphans.Count == 0)
4948
{
5049
Console.WriteLine("Dev Proxy is not running.");
5150
return 1;
@@ -61,9 +60,55 @@ private async Task<int> RunAsync(ParseResult parseResult, CancellationToken canc
6160
}
6261
}
6362

63+
ReportReconciledOrphans(reconciliation);
64+
6465
return exitCode;
6566
}
6667

68+
private static async Task<int> StopByPidAsync(int pid, bool force, CancellationToken cancellationToken)
69+
{
70+
// Capture a potential orphaned system-proxy record before LoadStateByPidAsync
71+
// prunes (deletes) the stale state file for a dead PID.
72+
var orphan = (await StateManager.GetOrphanedSystemProxyStatesAsync(cancellationToken))
73+
.Find(o => o.Pid == pid);
74+
75+
var state = await StateManager.LoadStateByPidAsync(pid, cancellationToken);
76+
if (state is not null)
77+
{
78+
return await StopInstanceAsync(state, force, cancellationToken);
79+
}
80+
81+
if (orphan is not null)
82+
{
83+
var liveOwner = await StateManager.FindSystemProxyInstanceAsync(cancellationToken);
84+
if (liveOwner is null)
85+
{
86+
SystemProxyManager.Disable();
87+
Console.WriteLine($"Restored system proxy left by crashed Dev Proxy (PID: {pid}).");
88+
}
89+
else
90+
{
91+
Console.WriteLine($"Removed stale record for crashed Dev Proxy (PID: {pid}); system proxy is owned by a running instance (PID: {liveOwner.Pid}).");
92+
}
93+
94+
await StateManager.DeleteStateAsync(pid, cancellationToken);
95+
return 0;
96+
}
97+
98+
Console.WriteLine($"No running Dev Proxy instance with PID {pid}.");
99+
return 1;
100+
}
101+
102+
private static void ReportReconciledOrphans(SystemProxyManager.OrphanReconciliation reconciliation)
103+
{
104+
foreach (var orphan in reconciliation.Orphans)
105+
{
106+
Console.WriteLine(reconciliation.SystemProxyDisabled
107+
? $"Restored system proxy left by crashed Dev Proxy (PID: {orphan.Pid})."
108+
: $"Removed stale record for crashed Dev Proxy (PID: {orphan.Pid}); system proxy is owned by a running instance.");
109+
}
110+
}
111+
67112
private static async Task<int> StopInstanceAsync(ProxyInstanceState state, bool force, CancellationToken cancellationToken)
68113
{
69114
if (force)
@@ -146,7 +191,12 @@ private static async Task<int> StopInstanceAsync(ProxyInstanceState state, bool
146191

147192
private static async Task<int> ForceStopAsync(ProxyInstanceState state, CancellationToken cancellationToken)
148193
{
149-
DisableSystemProxy();
194+
// A killed process can't run its own cleanup, so restore the system proxy
195+
// on its behalf before terminating it.
196+
if (state.AsSystemProxy)
197+
{
198+
SystemProxyManager.Disable();
199+
}
150200

151201
try
152202
{
@@ -172,46 +222,4 @@ private static async Task<int> ForceStopAsync(ProxyInstanceState state, Cancella
172222
await StateManager.DeleteStateAsync(state.Pid, cancellationToken);
173223
return 0;
174224
}
175-
176-
/// <summary>
177-
/// Disables the system proxy on macOS by calling toggle-proxy.sh off.
178-
/// This ensures the system proxy settings are cleaned up even when the
179-
/// daemon process is killed forcefully (SIGKILL cannot be caught).
180-
/// </summary>
181-
private static void DisableSystemProxy()
182-
{
183-
if (!OperatingSystem.IsMacOS())
184-
{
185-
return;
186-
}
187-
188-
var bashScriptPath = Path.Join(AppContext.BaseDirectory, "toggle-proxy.sh");
189-
if (!File.Exists(bashScriptPath))
190-
{
191-
return;
192-
}
193-
194-
var startInfo = new ProcessStartInfo
195-
{
196-
FileName = "/bin/bash",
197-
Arguments = $"{bashScriptPath} off",
198-
RedirectStandardOutput = true,
199-
UseShellExecute = false,
200-
CreateNoWindow = true
201-
};
202-
203-
try
204-
{
205-
using var process = new Process { StartInfo = startInfo };
206-
process.Start();
207-
if (!process.WaitForExit(TimeSpan.FromSeconds(10)))
208-
{
209-
process.Kill();
210-
}
211-
}
212-
catch
213-
{
214-
// Best-effort cleanup — don't block the stop flow
215-
}
216-
}
217225
}

‎DevProxy/Program.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,26 @@ static async Task<int> StartDetachedProcessAsync(string[] args)
5151
{
5252
var isJsonOutput = IsJsonOutputRequested(args);
5353

54+
// Recover from a system-proxy registration left behind by a previous
55+
// instance that crashed before restoring the OS proxy. This must run before
56+
// any state-loading call below, which prunes (deletes) the stale state files
57+
// it depends on without restoring the proxy.
58+
var reconciliation = await SystemProxyManager.ReconcileOrphanedSystemProxiesAsync();
59+
foreach (var orphan in reconciliation.Orphans)
60+
{
61+
var message = reconciliation.SystemProxyDisabled
62+
? $"Recovered system proxy left by a crashed Dev Proxy instance (PID: {orphan.Pid})."
63+
: $"Removed stale system-proxy registration left by a crashed Dev Proxy instance (PID: {orphan.Pid}) while keeping the current system proxy owner unchanged.";
64+
if (isJsonOutput)
65+
{
66+
await Console.Out.WriteLineAsync(FormatJsonLogEntry("info", message));
67+
}
68+
else
69+
{
70+
await Console.Out.WriteLineAsync(message);
71+
}
72+
}
73+
5474
// Check if an instance is already running as system proxy
5575
var systemProxyInstance = await StateManager.FindSystemProxyInstanceAsync();
5676
if (systemProxyInstance is not null)

‎DevProxy/Proxy/ProxyEngine.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,26 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
159159
_logger.LogInformation("Dev Proxy listening on {IPAddress}:{Port}...", endPoint.IpAddress, endPoint.Port);
160160
}
161161

162+
// Recover from a system-proxy registration left behind by a previous
163+
// instance that crashed before restoring the OS proxy, so a stale
164+
// registration doesn't linger across runs.
165+
var reconciliation = await SystemProxyManager.ReconcileOrphanedSystemProxiesAsync(stoppingToken);
166+
foreach (var orphan in reconciliation.Orphans)
167+
{
168+
if (reconciliation.SystemProxyDisabled)
169+
{
170+
_logger.LogInformation(
171+
"Recovered system proxy left by a crashed Dev Proxy instance (PID: {Pid}).",
172+
orphan.Pid);
173+
}
174+
else
175+
{
176+
_logger.LogInformation(
177+
"Removed stale system-proxy registration left by a crashed Dev Proxy instance (PID: {Pid}) while keeping the current system proxy owner unchanged.",
178+
orphan.Pid);
179+
}
180+
}
181+
162182
if (_config.AsSystemProxy)
163183
{
164184
if (RunTime.IsWindows)

‎DevProxy/Proxy/SystemProxyManager.cs‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using DevProxy.Abstractions.Utils;
6+
using DevProxy.State;
7+
using System.Diagnostics;
8+
using Titanium.Web.Proxy;
9+
10+
namespace DevProxy.Proxy;
11+
12+
/// <summary>
13+
/// Restores the operating system proxy settings out-of-process.
14+
/// Used to recover from a Dev Proxy instance that registered itself as the
15+
/// system proxy but was terminated without running its normal cleanup (crash,
16+
/// SIGKILL, OOM, power loss). The operation is best-effort, idempotent, and
17+
/// safe to call even when no system proxy is currently configured.
18+
/// </summary>
19+
internal static class SystemProxyManager
20+
{
21+
/// <summary>
22+
/// The outcome of reconciling orphaned system-proxy registrations.
23+
/// </summary>
24+
/// <param name="Orphans">The orphaned instance records that were reconciled.</param>
25+
/// <param name="SystemProxyDisabled">
26+
/// True if the OS system proxy was disabled as part of reconciliation. This
27+
/// is false when a live instance still owns the system proxy, in which case
28+
/// only the stale state records are removed.
29+
/// </param>
30+
internal readonly record struct OrphanReconciliation(
31+
IReadOnlyList<ProxyInstanceState> Orphans,
32+
bool SystemProxyDisabled);
33+
34+
/// <summary>
35+
/// Disables the operating system proxy.
36+
/// On Windows this clears the WinINET system proxy settings; on macOS it
37+
/// runs <c>toggle-proxy.sh off</c>. On Linux this is a no-op because Dev
38+
/// Proxy never configures a system proxy there.
39+
/// </summary>
40+
public static void Disable()
41+
{
42+
try
43+
{
44+
if (OperatingSystem.IsWindows())
45+
{
46+
DisableWindows();
47+
}
48+
else if (OperatingSystem.IsMacOS())
49+
{
50+
DisableMacOS();
51+
}
52+
// Linux: Dev Proxy never sets a system proxy, so there's nothing to restore.
53+
}
54+
catch
55+
{
56+
// Best-effort cleanup — never block the stop flow.
57+
}
58+
}
59+
60+
/// <summary>
61+
/// Reconciles system-proxy registrations left behind by crashed instances.
62+
/// Restores the OS proxy (unless a live instance still owns it) and removes
63+
/// the stale state records. Safe to call when there are no orphans.
64+
/// </summary>
65+
public static async Task<OrphanReconciliation> ReconcileOrphanedSystemProxiesAsync(CancellationToken cancellationToken = default)
66+
{
67+
// Capture orphans before any liveness-pruning call deletes their state files.
68+
var orphans = await StateManager.GetOrphanedSystemProxyStatesAsync(cancellationToken);
69+
if (orphans.Count == 0)
70+
{
71+
return new([], false);
72+
}
73+
74+
// Only touch the global OS proxy setting if no live instance currently
75+
// owns it — otherwise we'd disable a proxy a running instance depends on.
76+
var liveOwner = await StateManager.FindSystemProxyInstanceAsync(cancellationToken);
77+
var disabled = false;
78+
if (liveOwner is null)
79+
{
80+
Disable();
81+
disabled = true;
82+
}
83+
84+
foreach (var orphan in orphans)
85+
{
86+
await StateManager.DeleteStateAsync(orphan.Pid, cancellationToken);
87+
}
88+
89+
return new(orphans, disabled);
90+
}
91+
92+
private static void DisableWindows()
93+
{
94+
// DisableAllSystemProxies clears the WinINET proxy settings directly and
95+
// does not require a running proxy, so a fresh instance is enough to undo
96+
// a registration left behind by a crashed process.
97+
using var proxyServer = new ProxyServer(userTrustRootCertificate: false);
98+
proxyServer.DisableAllSystemProxies();
99+
}
100+
101+
private static void DisableMacOS()
102+
{
103+
var bashScriptPath = Path.Join(ProxyUtils.AppFolder ?? AppContext.BaseDirectory, "toggle-proxy.sh");
104+
if (!File.Exists(bashScriptPath))
105+
{
106+
return;
107+
}
108+
109+
var startInfo = new ProcessStartInfo
110+
{
111+
FileName = "/bin/bash",
112+
RedirectStandardOutput = true,
113+
UseShellExecute = false,
114+
CreateNoWindow = true
115+
};
116+
startInfo.ArgumentList.Add(bashScriptPath);
117+
startInfo.ArgumentList.Add("off");
118+
119+
using var process = new Process { StartInfo = startInfo };
120+
_ = process.Start();
121+
if (!process.WaitForExit(TimeSpan.FromSeconds(10)))
122+
{
123+
process.Kill();
124+
}
125+
}
126+
}

0 commit comments

Comments
 (0)