Merge upstream #2174: Add prioritized render scheduling

OpenUtau.Core/Commands/Notifications.cs

@@ -237,7 +237,14 @@
     }
 
     public class PreRenderNotification : UNotification {
-        public override string ToString() => $"Pre-render notification.";
+        public readonly int focusTick;
+
+        public PreRenderNotification(UPart part = null, int focusTick = -1) {
+            this.part = part;
+            this.focusTick = focusTick;
+        }
+
+        public override string ToString() => "Pre-render notification.";
     }
 
     public class PartRenderedNotification : UNotification {

OpenUtau.Core/PlaybackManager.cs

@@ -191,6 +191,8 @@ private PlaybackManager() {
         double startMs;
         public int StartTick => DocManager.Inst.Project.timeAxis.MsPosToTickPos(startMs);
         CancellationTokenSource renderCancellation;
+        UVoicePart preRenderFocusPart;
+        int preRenderFocusTick = -1;
 
         // Loop playback state
         private int loopStartTick = 0;
@@ -416,7 +418,10 @@ private void CheckFileWritable(string filePath) {
 
         void SchedulePreRender() {
             Log.Information("SchedulePreRender");
-            var engine = new RenderEngine(DocManager.Inst.Project);
+            var engine = new RenderEngine(
+                DocManager.Inst.Project,
+                focusPart: preRenderFocusPart,
+                focusTick: preRenderFocusTick);
             engine.PreRenderProject(ref renderCancellation);
         }
 
@@ -443,7 +448,24 @@ public void OnNext(UCommand cmd, bool isUndo) {
                 renderCancellation?.Cancel();
                 LiveWaveformCache.Clear();
                 DocManager.Inst.ExecuteCmd(new WaveformReadyNotification());
+                preRenderFocusPart = null;
+                preRenderFocusTick = -1;
                 DocManager.Inst.ExecuteCmd(new SetPlayPosTickNotification(0));
+            } else if (cmd is LoadPartNotification loadPart) {
+                preRenderFocusPart = loadPart.part as UVoicePart;
+                preRenderFocusTick = loadPart.tick;
+            } else if (cmd is FocusNoteNotification focusNote) {
+                preRenderFocusPart = focusNote.part as UVoicePart;
+                preRenderFocusTick = focusNote.part?.position + focusNote.note.position ?? preRenderFocusTick;
+            } else if (cmd is SetPlayPosTickNotification setPlayPosTick) {
+                preRenderFocusTick = setPlayPosTick.playPosTick;
+            } else if (cmd is PreRenderNotification preRender) {
+                if (preRender.part is UVoicePart voicePart) {
+                    preRenderFocusPart = voicePart;
+                }
+                if (preRender.focusTick >= 0) {
+                    preRenderFocusTick = preRender.focusTick;
+                }
             }
             if (cmd is PreRenderNotification || cmd is LoadProjectNotification) {
                 if (Util.Preferences.Default.PreRender) {

OpenUtau.Core/Render/RenderEngine.cs

@@ -47,15 +47,25 @@
         readonly int startTick;
         readonly int endTick;
         readonly int trackNo;
+        readonly UVoicePart focusPart;
+        readonly int focusTick;
 
         static readonly System.Collections.Concurrent.ConcurrentDictionary<string, float[]> XsyBlendCache =
             new System.Collections.Concurrent.ConcurrentDictionary<string, float[]>();
 
-        public RenderEngine(UProject project, int startTick = 0, int endTick = -1, int trackNo = -1) {
+        public RenderEngine(
+            UProject project,
+            int startTick = 0,
+            int endTick = -1,
+            int trackNo = -1,
+            UVoicePart focusPart = null,
+            int focusTick = -1) {
             this.project = project;
             this.startTick = startTick;
             this.endTick = endTick;
             this.trackNo = trackNo;
+            this.focusPart = focusPart;
+            this.focusTick = focusTick;
         }
 
         // for playback or export
@@ -244,21 +254,24 @@
             }
             var tuples = requests
                 .SelectMany(req => req.phrases
-                    .Zip(req.sources, (phrase, source) => Tuple.Create(phrase, source, req)))
+                    .Zip(req.sources, (phrase, source) => (phrase, source, request: req)))
                 .ToArray();
+            if (tuples.Length == 0) {
+                return;
+            }
             if (playing) {
-                var orderedTuples = tuples
-                    .Where(tuple => tuple.Item1.end > startTick)
-                    .OrderBy(tuple => tuple.Item1.end)
-                    .Concat(tuples.Where(tuple => tuple.Item1.end <= startTick))
-                    .ToArray();
-                tuples = orderedTuples;
+                tuples = OrderForPlayback(tuples);
+            } else if (focusPart != null || focusTick >= 0) {
+                tuples = OrderForPreRender(tuples);
             }
             var progress = new Progress(tuples.Sum(t => t.Item1.phones.Length));
             foreach (var tuple in tuples) {
-                var phrase = tuple.Item1;
-                var source = tuple.Item2;
-                var request = tuple.Item3;
+                if (cancellation.IsCancellationRequested) {
+                    break;
+                }
+                var phrase = tuple.phrase;
+                var source = tuple.source;
+                var request = tuple.request;
                 bool useXsy = phrase.xsy != null && phrase.xsy.Any(x => x > 0);
                 if (!useXsy) {
                     var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true);
@@ -341,6 +354,50 @@
             progress.Clear();
         }
 
+        private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPlayback(
+            (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) {
+            double playbackStartMs = project.timeAxis.TickPosToMsPos(startTick);
+            return tuples
+                .Select((tuple, index) => (tuple, index))
+                .OrderBy(item => RenderPriority.PlaybackBucket(
+                    item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs))
+                .ThenBy(item => RenderPriority.PlaybackDistance(
+                    item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs))
+                .ThenBy(item => item.index)
+                .Select(item => item.tuple)
+                .ToArray();
+        }
+
+        private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPreRender(
+            (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) {
+            return tuples
+                .Select((tuple, index) => (tuple, index))
+                .OrderBy(item => PreRenderAttentionBucket(item.tuple))
+                .ThenBy(item => PreRenderAttentionDistance(item.tuple.phrase))
+                .ThenBy(item => item.index)
+                .Select(item => item.tuple)
+                .ToArray();
+        }
+
+        private int PreRenderAttentionBucket(
+            (RenderPhrase phrase, WaveSource source, RenderPartRequest request) tuple) {
+            bool isPriorityPart = focusPart != null && ReferenceEquals(tuple.request.part, focusPart);
+            bool overlapsPriority = focusTick >= 0 &&
+                tuple.phrase.position <= focusTick &&
+                tuple.phrase.end > focusTick;
+            bool isAfterPriorityStart = focusTick < 0 || tuple.phrase.end > focusTick;
+            return RenderPriority.PreRenderBucket(
+                isPriorityPart,
+                overlapsPriority,
+                isAfterPriorityStart);
+        }
+
+        private int PreRenderAttentionDistance(RenderPhrase phrase) {
+            return focusTick >= 0
+                ? RenderPriority.PreRenderDistance(phrase.position, phrase.end, focusTick)
+                : 0;
+        }
+
         public static void ReleaseSourceTemp() {
             VoicebankFiles.Inst.ReleaseSourceTemp();
         }

OpenUtau.Core/Render/RenderPriority.cs

@@ -0,0 +1,48 @@
+using System;
+
+namespace OpenUtau.Core.Render {
+    internal static class RenderPriority {
+        internal static int PlaybackBucket(double sourceStartMs, double sourceEndMs, double playbackStartMs) {
+            if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) {
+                return 0;
+            }
+            return sourceStartMs >= playbackStartMs ? 1 : 2;
+        }
+
+        internal static double PlaybackDistance(double sourceStartMs, double sourceEndMs, double playbackStartMs) {
+            if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) {
+                return Math.Max(0, playbackStartMs - sourceStartMs);
+            }
+            if (sourceStartMs >= playbackStartMs) {
+                return sourceStartMs - playbackStartMs;
+            }
+            return playbackStartMs - sourceEndMs;
+        }
+
+        internal static int PreRenderBucket(
+            bool isPriorityPart,
+            bool overlapsPriority,
+            bool isAfterPriorityStart) {
+            if (isPriorityPart && overlapsPriority) {
+                return 0;
+            }
+            if (isPriorityPart) {
+                return 1;
+            }
+            if (isAfterPriorityStart) {
+                return 2;
+            }
+            return 3;
+        }
+
+        internal static int PreRenderDistance(int phraseStartTick, int phraseEndTick, int priorityStartTick) {
+            if (phraseStartTick <= priorityStartTick && phraseEndTick > priorityStartTick) {
+                return 0;
+            }
+            if (phraseStartTick >= priorityStartTick) {
+                return phraseStartTick - priorityStartTick;
+            }
+            return priorityStartTick - phraseEndTick;
+        }
+    }
+}

OpenUtau.Test/Core/Render/RenderPriorityTest.cs

@@ -0,0 +1,38 @@
+using Xunit;
+
+namespace OpenUtau.Core.Render {
+    public class RenderPriorityTest {
+        [Fact]
+        public void PlaybackBucket_PrioritizesCurrentThenFutureThenPast() {
+            Assert.Equal(0, RenderPriority.PlaybackBucket(100, 200, 150));
+            Assert.Equal(1, RenderPriority.PlaybackBucket(200, 300, 150));
+            Assert.Equal(2, RenderPriority.PlaybackBucket(0, 100, 150));
+        }
+
+        [Fact]
+        public void PlaybackDistance_PrioritizesEarlierOffsetInCurrentBucket() {
+            Assert.Equal(50, RenderPriority.PlaybackDistance(100, 200, 150));
+            Assert.Equal(50, RenderPriority.PlaybackDistance(200, 300, 150));
+            Assert.Equal(50, RenderPriority.PlaybackDistance(0, 100, 150));
+        }
+
+        [Fact]
+        public void PreRenderBucket_PrioritizesFocusedPartAtAttentionTick() {
+            Assert.Equal(0, RenderPriority.PreRenderBucket(
+                isPriorityPart: true, overlapsPriority: true, isAfterPriorityStart: true));
+            Assert.Equal(1, RenderPriority.PreRenderBucket(
+                isPriorityPart: true, overlapsPriority: false, isAfterPriorityStart: true));
+            Assert.Equal(2, RenderPriority.PreRenderBucket(
+                isPriorityPart: false, overlapsPriority: false, isAfterPriorityStart: true));
+            Assert.Equal(3, RenderPriority.PreRenderBucket(
+                isPriorityPart: false, overlapsPriority: false, isAfterPriorityStart: false));
+        }
+
+        [Fact]
+        public void PreRenderDistance_PrioritizesPhraseContainingAttentionTick() {
+            Assert.Equal(0, RenderPriority.PreRenderDistance(100, 200, 150));
+            Assert.Equal(50, RenderPriority.PreRenderDistance(200, 300, 150));
+            Assert.Equal(50, RenderPriority.PreRenderDistance(0, 100, 150));
+        }
+    }
+}