web/queue: replace pipelineResults array with object

web/src/components/queue/ProcessingQueueItem.svelte

@@ -96,7 +96,8 @@
             // if only fetch workers are running, then we should
             // show the sum of all running & completed fetch workers
             if (running.size === 1 && running.has("fetch")) {
-                totalSize += info.pipelineResults.reduce((s, p) => s + (p?.size ?? 0), 0);
+                totalSize += Object.values(info.pipelineResults)
+                                   .reduce((s, p) => s + (p?.size ?? 0), 0);
             }
 
             const runningText = [...running].map(task => $t(`queue.state.running.${task}`)).join(", ");
@@ -107,7 +108,7 @@
             }
 
             const firstUnstarted = info.pipeline.find(w => {
-                if (info.completedWorkers.has(w.workerId))
+                if (info.pipelineResults[w.workerId])
                     return false;
 
                 const task = currentTasks[w.workerId];
@@ -134,7 +135,7 @@
     };
 
     const getWorkerProgress = (item: CobaltQueueItem, workerId: UUID): number | undefined => {
-        if (item.state === 'running' && item.completedWorkers.has(workerId)) {
+        if (item.state === 'running' && item.pipelineResults[workerId]) {
             return 100;
         }
 
@@ -187,7 +188,7 @@
                     <ProgressBar
                         percentage={getWorkerProgress(info, task.workerId) || 0}
                         workerId={task.workerId}
-                        completedWorkers={info.completedWorkers}
+                        pipelineResults={info.pipelineResults}
                     />
                 {/each}
             </div>

web/src/components/queue/ProgressBar.svelte

@@ -1,14 +1,14 @@
 <script lang="ts">
     import Skeleton from "$components/misc/Skeleton.svelte";
-    import type { UUID } from "$lib/types/queue";
+    import type { CobaltQueueItemRunning, UUID } from "$lib/types/queue";
 
     type Props = {
         percentage?: number;
         workerId: UUID;
-        completedWorkers: Set<string>;
+        pipelineResults: CobaltQueueItemRunning['pipelineResults'];
     }
 
-    let { percentage = 0, workerId, completedWorkers }: Props = $props();
+    let { percentage = 0, workerId, pipelineResults }: Props = $props();
 </script>
 
 <div class="file-progress">
@@ -17,7 +17,7 @@
             class="progress"
             style="width: {Math.min(100, percentage)}%"
         ></div>
-    {:else if completedWorkers.has(workerId)}
+    {:else if pipelineResults[workerId]}
         <div
             class="progress"
             style="width: 100%"

web/src/lib/state/task-manager/queue.ts

@@ -6,11 +6,15 @@ import { clearCurrentTasks, removeWorkerFromQueue } from "$lib/state/task-manage
 
 import type { CobaltQueue, CobaltQueueItem, CobaltQueueItemRunning, UUID } from "$lib/types/queue";
 
+export const DUMMY_FILE = new File([], 'pipeline_result_deleted.bin');
+
 const clearPipelineCache = (queueItem: CobaltQueueItem) => {
     if (queueItem.state === "running") {
-        let item: File | undefined;
+        for (const [ workerId, item ] of Object.entries(queueItem.pipelineResults)) {
-        while ((item = queueItem.pipelineResults.pop())) {
+            if (item.name !== DUMMY_FILE.name) {
                 removeFromFileStorage(item.name);
+                queueItem.pipelineResults[workerId] = DUMMY_FILE;
+            }
         }
     } else if (queueItem.state === "done") {
         removeFromFileStorage(queueItem.resultFile.name);
@@ -75,8 +79,7 @@ export function pipelineTaskDone(id: UUID, workerId: UUID, file: File) {
         const item = queueData[id];
 
         if (item && item.state === 'running') {
-            item.pipelineResults.push(file);
+            item.pipelineResults[workerId] = file;
-            item.completedWorkers.add(workerId);
         }
 
         return queueData;
@@ -92,8 +95,7 @@ export function itemRunning(id: UUID) {
 
         if (data) {
             data.state = 'running';
-            data.completedWorkers ??= new Set();
+            data.pipelineResults ??= {};
-            data.pipelineResults ??= [];
         }
 
         return queueData;

web/src/lib/task-manager/queue.ts

@@ -217,7 +217,7 @@ export const getProgress = (item: CobaltQueueItem, currentTasks: CobaltCurrentTa
 
     let sum = 0;
     for (const worker of item.pipeline) {
-        if (item.completedWorkers.has(worker.workerId)) {
+        if (item.pipelineResults[worker.workerId]) {
             sum += 1;
         } else {
             const task = currentTasks[worker.workerId];

web/src/lib/task-manager/run-worker.ts

@@ -12,7 +12,7 @@ export const killWorker = (worker: Worker, unsubscribe: () => void, interval?: N
     if (interval) clearInterval(interval);
 }
 
-export const startWorker = async ({ worker, workerId, parentId, workerArgs }: CobaltPipelineItem) => {
+export const startWorker = async ({ worker, workerId, dependsOn, parentId, workerArgs }: CobaltPipelineItem) => {
     let files: File[] = [];
 
     switch (worker) {
@@ -22,10 +22,15 @@ export const startWorker = async ({ worker, workerId, parentId, workerArgs }: Co
                 files = workerArgs.files;
             }
 
-            if (files.length === 0) {
             const parent = get(queue)[parentId];
-                if (parent.state === "running" && parent.pipelineResults.length) {
+            if (parent?.state === "running" && dependsOn) {
-                    files = parent.pipelineResults;
+                for (const workerId of dependsOn) {
+                    const file = parent.pipelineResults[workerId];
+                    if (!file) {
+                        return itemError(parentId, workerId, "queue.ffmpeg.no_args");
+                    }
+
+                    files.push(file);
                 }
             }
 

web/src/lib/task-manager/scheduler.ts

@@ -1,7 +1,7 @@
 import { get } from "svelte/store";
 import { startWorker } from "$lib/task-manager/run-worker";
 import { addWorkerToQueue, currentTasks } from "$lib/state/task-manager/current-tasks";
-import { itemDone, itemError, itemRunning, queue } from "$lib/state/task-manager/queue";
+import { DUMMY_FILE, itemDone, itemError, itemRunning, queue } from "$lib/state/task-manager/queue";
 
 import type { CobaltPipelineItem } from "$lib/types/workers";
 
@@ -27,8 +27,11 @@ export const schedule = () => {
 
             // if all workers are completed, then return the
             // the final file and go to the next task
-            if (task.completedWorkers.size === task.pipeline.length) {
+            if (Object.keys(task.pipelineResults).length === task.pipeline.length) {
-                const finalFile = task.pipelineResults.pop();
+                // swap final file for a dummy, so that it doesn't get
+                // deleted when we clean up the intermediate files
+                const finalFile = task.pipelineResults[finalWorker.workerId];
+                task.pipelineResults[finalWorker.workerId] = DUMMY_FILE;
 
                 if (finalFile) {
                     itemDone(task.id, finalFile);
@@ -42,11 +45,11 @@ export const schedule = () => {
             // if current worker is completed, but there are more workers,
             // then start the next one and wait to be called again
             for (const worker of task.pipeline) {
-                if (task.completedWorkers.has(worker.workerId) || ongoingTasks[worker.workerId]) {
+                if (task.pipelineResults[worker.workerId] || ongoingTasks[worker.workerId]) {
                     continue;
                 }
 
-                const needsToWait = worker.dependsOn?.some(id => !task.completedWorkers.has(id));
+                const needsToWait = worker.dependsOn?.some(id => !task.pipelineResults[id]);
                 if (needsToWait) {
                     break;
                 }

web/src/lib/types/queue.ts

@@ -19,8 +19,7 @@ type CobaltQueueItemWaiting = CobaltQueueBaseItem & {
 
 export type CobaltQueueItemRunning = CobaltQueueBaseItem & {
     state: "running",
-    completedWorkers: Set<UUID>,
+    pipelineResults: Record<UUID, File>,
-    pipelineResults: File[],
 };
 
 type CobaltQueueItemDone = CobaltQueueBaseItem & {