Skip to content

Implement XSY cross-synthesis for classic rendering #2261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
emeraldsingers wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Implement XSY cross-synthesis for classic rendering #2261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
461535c
Implement XSY cross-synthesis for classic rendering
emeraldsingers Jul 12, 2026
fe780da
Merge branch 'openutau:master' into XSY
emeraldsingers Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions OpenUtau.Core/Classic/ClassicRenderer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public class ClassicRenderer : IRenderer {
Ustx.DYN,
Ustx.PITD,
Ustx.CLR,
Ustx.XSYC,
Ustx.XSY,
Ustx.ENG,
Ustx.VEL,
Ustx.VOL,
Expand Down
2 changes: 2 additions & 0 deletions OpenUtau.Core/Classic/WorldlineRenderer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public WorldlineRenderer(int version) {
Ustx.DYN,
Ustx.PITD,
Ustx.CLR,
Ustx.XSYC,
Ustx.XSY,
Ustx.SHFT,
Ustx.VEL,
Ustx.VOL,
Expand Down
4 changes: 4 additions & 0 deletions OpenUtau.Core/Format/USTx.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class Ustx {
public const string SHFC = "shfc";
public const string TENC = "tenc";
public const string VOIC = "voic";
public const string XSYC = "xsyc";
public const string XSY = "xsy";

public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC };

Expand All @@ -60,6 +62,8 @@ public static void AddDefaultExpressions(UProject project) {
project.RegisterExpression(new UExpressionDescriptor("tone shift (curve)", SHFC, -1200, 1200, 0) { type = UExpressionType.Curve });
project.RegisterExpression(new UExpressionDescriptor("tension (curve)", TENC, -100, 100, 0) { type = UExpressionType.Curve });
project.RegisterExpression(new UExpressionDescriptor("voicing (curve)", VOIC, 0, 100, 100) { type = UExpressionType.Curve });
project.RegisterExpression(new UExpressionDescriptor("cross synthesis color", XSYC, false, new string[0]));
project.RegisterExpression(new UExpressionDescriptor("cross synthesis (curve)", XSY, 0, 100, 0) { type = UExpressionType.Curve });

string message = string.Empty;
if (ValidateExpression(project, "g", GEN)) {
Expand Down
81 changes: 76 additions & 5 deletions OpenUtau.Core/Render/RenderEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ class RenderEngine {
readonly int endTick;
readonly int trackNo;

static readonly System.Collections.Concurrent.ConcurrentDictionary<string, float[]> XsyBlendCache =
new System.Collections.Concurrent.ConcurrentDictionary<string, float[]>();

public RenderEngine(UProject project, int startTick = 0, int endTick = -1, int trackNo = -1) {
this.project = project;
this.startTick = startTick;
Expand Down Expand Up @@ -253,12 +256,80 @@ private void RenderRequests(
var phrase = tuple.Item1;
var source = tuple.Item2;
var request = tuple.Item3;
var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true);
task.Wait();
if (cancellation.IsCancellationRequested) {
break;
bool useXsy = phrase.xsy != null && phrase.xsy.Any(x => x > 0);
if (!useXsy) {
var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true);
task.Wait();
if (cancellation.IsCancellationRequested) {
break;
}
source.SetSamples(task.Result.samples);
} else {
string xsyKey = $"{phrase.hash:x16}|" +
string.Join(",", phrase.phones.Select(p => $"{p.oto2?.Set}:{p.oto2?.Alias}"));
if (!XsyBlendCache.TryGetValue(xsyKey, out var blended)) {
var taskA = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true);
taskA.Wait();
if (cancellation.IsCancellationRequested) {
break;
}
float[] samplesA = taskA.Result.samples;

var otoField = typeof(RenderPhone).GetField("oto");
var hashField = typeof(RenderPhone).GetField("hash");
var phraseHashField = typeof(RenderPhrase).GetField("hash");
var originalOtos = phrase.phones.Select(p => p.oto).ToArray();
var originalHashes = phrase.phones.Select(p => p.hash).ToArray();
ulong originalPhraseHash = phrase.hash;
float[] samplesB;
try {
for (int i = 0; i < phrase.phones.Length; i++) {
var phone = phrase.phones[i];
if (phone.oto2 != null) {
otoField.SetValue(phone, phone.oto2);
hashField.SetValue(phone, phone.hash ^ 0x5858585858585858);
}
}
phraseHashField.SetValue(phrase, phrase.hash ^ 0x5858585858585858);
var taskB = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true);
taskB.Wait();
samplesB = taskB.Result.samples;
} finally {
for (int i = 0; i < phrase.phones.Length; i++) {
otoField.SetValue(phrase.phones[i], originalOtos[i]);
hashField.SetValue(phrase.phones[i], originalHashes[i]);
}
phraseHashField.SetValue(phrase, originalPhraseHash);
}
if (cancellation.IsCancellationRequested) {
break;
}

const int fftSize = 2048;
const int hopSize = 512;
int totalSamples = Math.Max(samplesA.Length, samplesB.Length);
int frameCount = Math.Max(1, (totalSamples - fftSize) / hopSize + 1);
float[] frameRatios = new float[frameCount];
int pitchStart = phrase.position - phrase.leading;
for (int f = 0; f < frameCount; f++) {
double timeMs = phrase.positionMs - phrase.leadingMs
+ (double)(f * hopSize) / 44100.0 * 1000.0;
double tick = project.timeAxis.MsPosToTickPos(timeMs);
int curveIndex = (int)Math.Max(0, (tick - pitchStart) / 5);
if (phrase.xsy.Length > 0) {
frameRatios[f] = curveIndex < phrase.xsy.Length
? Math.Clamp(phrase.xsy[curveIndex] / 100f, 0f, 1f)
: Math.Clamp(phrase.xsy.Last() / 100f, 0f, 1f);
}
}
blended = CrossSynthDSP.StftBlend(samplesA, samplesB, frameRatios);
if (XsyBlendCache.Count > 1024) {
XsyBlendCache.Clear();
}
XsyBlendCache[xsyKey] = blended;
}
source.SetSamples(blended);
}
source.SetSamples(task.Result.samples);
if (request.sources.All(s => s.HasSamples)) {
request.part.SetMix(request.mix);
DocManager.Inst.ExecuteCmd(new PartRenderedNotification(request.part));
Expand Down
25 changes: 24 additions & 1 deletion OpenUtau.Core/Render/RenderPhrase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public class RenderPhone {
public readonly double adjustedTempo;
public readonly Tuple<string, int?, string>[] flags;// flag, value, abbr. Abbr is kept here for flag filtering.
public readonly string suffix;
public readonly string suffix2;
public readonly float volume;
public readonly float velocity;
public readonly float modulation;
Expand All @@ -76,6 +77,7 @@ public class RenderPhone {
public readonly int toneShift;

public readonly UOto oto;
public readonly UOto oto2;
public readonly ulong hash;

internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note, UPhoneme phoneme, int phrasePosition) {
Expand Down Expand Up @@ -120,6 +122,9 @@ internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note
string voiceColor = phoneme.GetVoiceColor(project, track);
suffix = track.Singer.Subbanks.FirstOrDefault(
subbank => subbank.Color == voiceColor)?.Suffix ?? string.Empty;
string targetColor = phoneme.GetVoiceColor2(project, track) ?? string.Empty;
suffix2 = track.Singer.Subbanks.FirstOrDefault(
subbank => subbank.Color == targetColor)?.Suffix ?? string.Empty;
volume = phoneme.GetExpression(project, track, Format.Ustx.VOL).Item1 * 0.01f;
velocity = phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 * 0.01f;
modulation = phoneme.GetExpression(project, track, Format.Ustx.MOD).Item1 * 0.01f;
Expand All @@ -129,6 +134,12 @@ internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note
toneShift = (int)phoneme.GetExpression(project, track, Format.Ustx.SHFT).Item1;

oto = phoneme.oto;
if (oto != null && !string.IsNullOrEmpty(targetColor)) {
string basePhoneme = oto.Phonetic ?? phoneme.phoneme;
if (track.Singer.TryGetMappedOto(basePhoneme, note.tone, targetColor, out var secondaryOto)) {
oto2 = secondaryOto;
}
}
hash = Hash();
}
private ulong Hash() {
Expand All @@ -147,6 +158,7 @@ private ulong Hash() {
}
}
writer.Write(suffix);
writer.Write(suffix2 ?? string.Empty);
writer.Write(volume);
writer.Write(velocity);
writer.Write(modulation);
Expand Down Expand Up @@ -187,6 +199,7 @@ public class RenderPhrase {
public readonly float[] toneShift;
public readonly float[] tension;
public readonly float[] voicing;
public readonly float[] xsy;
public readonly Tuple<string, float[]>[] curves;//custom curves defined by renderer
public readonly ulong preEffectHash;
public readonly ulong hash;
Expand Down Expand Up @@ -442,6 +455,16 @@ double Fade(double diff, int pit) {
case Format.Ustx.TENC: tension = curveSampled; break;
case Format.Ustx.BREC: breathiness = curveSampled; break;
case Format.Ustx.VOIC: voicing = curveSampled; break;
case Format.Ustx.XSY:
xsy = curveSampled;
foreach (var phone in phones) {
int startIdx = Math.Max(0, (phone.position - phone.leading - pitchStart) / pitchInterval);
int endIdx = Math.Min(xsy.Length, Math.Max(0, (phone.position - pitchStart) / pitchInterval));
for (int k = startIdx; k < endIdx; k++) {
xsy[k] = 0f;
}
}
break;
default:
curves.Add(Tuple.Create(curve.abbr,curveSampled));
break;
Expand Down Expand Up @@ -498,7 +521,7 @@ private ulong Hash(bool postEffect) {
writer.Write(phone.hash);
}
if (postEffect) {
foreach (var array in new float[][] { pitches, dynamics, gender, breathiness, toneShift, tension, voicing }) {
foreach (var array in new float[][] { pitches, dynamics, gender, breathiness, toneShift, tension, voicing, xsy }) {
if (array == null) {
writer.Write("null");
} else {
Expand Down
Loading
Loading