Handle shutdowns better, allow retrying multis

ALttPRandomizer/Model/SeedSettings.cs

@@ -42,6 +42,10 @@
         [ForbiddenSetting([Apr2025], EntranceShuffle.Swapped)]
         public EntranceShuffle EntranceShuffle { get; set; } = EntranceShuffle.Vanilla;
 
+        [NoSettingName]
+        [RequiredSetting([Apr2025], LinksHouse.Vanilla)]
+        public LinksHouse LinksHouse { get; set; } = LinksHouse.Vanilla;
+
         [SettingName("skullwoods")]
         [RequiredSetting([Apr2025], SkullWoodsShuffle.Original)]
         [NoSettingName([Apr2025])]
@@ -250,6 +254,11 @@
         [SettingName("insanity")] Decoupled,
     }
 
+    public enum LinksHouse {
+        Vanilla,
+        [AdditionalSetting("--shufflelinks")] Shuffled,
+    }
+
     public enum SkullWoodsShuffle {
         Original,
         Restricted,

ALttPRandomizer/Program.cs

@@ -69,6 +69,7 @@
             builder.Services.AddSingleton(sp => sp);
             builder.Services.AddSingleton<AzureStorage>();
             builder.Services.AddSingleton<CommonSettingsProcessor>();
+            builder.Services.AddSingleton<ShutdownHandler>();
 
             builder.Services.AddKeyedScoped<IRandomizer, BaseRandomizer>(BaseRandomizer.Name);
             builder.Services.AddKeyedScoped<IRandomizer, BaseRandomizer>(BaseRandomizer.DungeonMapName);
@@ -90,7 +91,11 @@
                 c.EnableValidator();
             });
 
+            var sh = app.Services.GetService<ShutdownHandler>();
+
             app.Run();
+
+            sh?.HandleShutdown().Wait();
         }
     }
 }

ALttPRandomizer/Randomizers/BaseRandomizer.cs

@@ -18,18 +18,20 @@
         public const string Name = "base";
         public const string DungeonMapName = "dungeon_map";
 
-        public const int MULTI_TRIES = 20;
+        public const int MULTI_TRIES = 80;
-        public const int SINGLE_TRIES = 20;
+        public const int SINGLE_TRIES = 5;
 
         public BaseRandomizer(
                 AzureStorage azureStorage,
                 CommonSettingsProcessor settingsProcessor,
                 IdGenerator idGenerator,
+                ShutdownHandler shutdownHandler,
                 IOptionsMonitor<ServiceOptions> optionsMonitor,
                 ILogger<BaseRandomizer> logger) {
             this.AzureStorage = azureStorage;
             this.SettingsProcessor = settingsProcessor;
             this.IdGenerator = idGenerator;
+            this.ShutdownHandler = shutdownHandler;
             this.OptionsMonitor = optionsMonitor;
             this.Logger = logger;
         }
@@ -40,6 +42,7 @@
         private IOptionsMonitor<ServiceOptions> OptionsMonitor { get; }
         private ILogger<BaseRandomizer> Logger { get; }
         private ServiceOptions Configuration => OptionsMonitor.CurrentValue;
+        private ShutdownHandler ShutdownHandler { get; }
 
         public void Validate(SeedSettings settings) {
             this.SettingsProcessor.ValidateSettings(settings.Randomizer, settings);
@@ -58,7 +61,6 @@
             var args = new List<string>() {
                 "--reduce_flashing",
                 "--quickswap",
-                "--shufflelinks",
                 "--shuffletavern",
             };
 
@@ -100,13 +102,11 @@
 
             args.Add("--spoiler=json");
 
-            args.Add(string.Format("--tries={0}", SINGLE_TRIES));
-
             foreach (var arg in settings) {
                 args.Add(arg);
             }
 
-            Logger.LogInformation("Randomizing with args: {args}", string.Join(" ", args));
+            Logger.LogInformation("Randomizing {id} with args: {args}", id, string.Join(" ", args.Select(arg => $"\"{arg}\"")));
 
             var generating = string.Format("{0}/generating", id);
             await AzureStorage.UploadFile(generating, BinaryData.Empty);
@@ -114,14 +114,17 @@
             var process = Process.Start(start) ?? throw new GenerationFailedException("Process failed to start.");
             process.EnableRaisingEvents = true;
 
-            process.OutputDataReceived += (_, args) => Logger.LogInformation("Randomizer STDOUT: {output}", args.Data);
+            process.OutputDataReceived += (_, args) => Logger.LogInformation("Randomizer {id} STDOUT: {output}", id, args.Data);
-            process.ErrorDataReceived += (_, args) => Logger.LogInformation("Randomizer STDERR: {output}", args.Data);
+            process.ErrorDataReceived += (_, args) => Logger.LogInformation("Randomizer {id} STDERR: {output}", id, args.Data);
 
             process.BeginOutputReadLine();
             process.BeginErrorReadLine();
 
+            this.ShutdownHandler.AddId(id);
+
             process.Exited += async (sender, args) => {
                 try {
+                    this.ShutdownHandler.RemoveId(id);
                     await completed.Invoke(process.ExitCode);
                 } catch (Exception ex) {
                     this.Logger.LogError(ex, "Error while invoking completion of randomizer generation.");
@@ -132,7 +135,9 @@
         public async Task Randomize(string id, SeedSettings settings) {
             Logger.LogDebug("Recieved request for id {id} to randomize settings {@settings}", id, settings);
 
-            await StartProcess(this.SettingsProcessor.GetRandomizerName(settings.Randomizer), id, this.GetArgs(settings), async exitcode => {
+            var args = this.GetArgs(settings).Append(string.Format("--tries={0}", SINGLE_TRIES));
+
+            await StartProcess(this.SettingsProcessor.GetRandomizerName(settings.Randomizer), id, args, async exitcode => {
                 if (exitcode != 0) {
                     await GenerationFailed(id, exitcode);
                 } else {

ALttPRandomizer/SeedController.cs

@@ -66,6 +66,12 @@
             return ResolveResult(await this.SeedService.GetMulti(id));
         }
 
+        [Route("/multi/{id}")]
+        [HttpPost]
+        public async Task<ActionResult> RetryMulti(string id) {
+            return ResolveResult(await this.RandomizeService.RetryMulti(id));
+        }
+
         private ActionResult ResolveResult(IDictionary<string, object> result) {
             if (result.TryGetValue("status", out var responseCode)) {
                 if (responseCode is int code) {

ALttPRandomizer/Service/RandomizeService.cs

@@ -95,5 +95,46 @@
             result["status"] = 202;
             return result;
         }
+
+        public async Task<IDictionary<string, object>> RetryMulti(string multiId) {
+            var files = await this.AzureStorage.GetFiles(multiId);
+
+            this.Logger.LogDebug("Found files: {@files}", files.Keys);
+
+            var result = new Dictionary<string, object>();
+
+            if (!files.TryGetValue("settings.json", out var settingsData)) {
+                result["status"] = 404;
+                result["error"] = "multi settings not found";
+                return result;
+            }
+
+            var settingsJson = JsonDocument.Parse(settingsData.ToString());
+
+            var settings = settingsJson.Deserialize<IList<SeedSettings>>(JsonOptions.Default);
+
+            if (settings == null) {
+                result["status"] = 404;
+                result["error"] = "multi settings not found";
+                return result;
+            }
+
+            if (files.TryGetValue("patch.bps", out var patchData)) {
+                result["status"] = 409;
+                result["error"] = "generation already successful";
+                return result;
+            }
+
+            if (files.ContainsKey("generating")) {
+                result["status"] = 409;
+                result["error"] = "generation still in progress";
+                return result;
+            }
+
+            await this.RandomizeMultiworld(settings, multiId);
+
+            result["status"] = 202;
+            return result;
+        }
     }
 }

ALttPRandomizer/Service/SeedService.cs

@@ -102,6 +102,7 @@
                 } else {
                     result["status"] = 404;
                     result["error"] = "generation failed";
+                    result["retry"] = true;
                     return result;
                 }
             }

ALttPRandomizer/ShutdownHandler.cs

@@ -0,0 +1,40 @@
+namespace ALttPRandomizer {
+    using System.Collections.Generic;
+    using System.Threading.Tasks;
+    using ALttPRandomizer.Azure;
+    using Microsoft.Extensions.Logging;
+
+    public class ShutdownHandler {
+        public ShutdownHandler(AzureStorage azureStorage, ILogger<ShutdownHandler> logger) {
+            this.AzureStorage = azureStorage;
+            this.Logger = logger;
+        }
+
+        private AzureStorage AzureStorage { get; }
+        private ILogger<ShutdownHandler> Logger { get; }
+
+        private HashSet<string> IdsInProgress { get; } = new();
+
+        public void AddId(string id) {
+            this.IdsInProgress.Add(id);
+        }
+
+        public void RemoveId(string id) {
+            this.IdsInProgress.Remove(id);
+        }
+
+        public async Task HandleShutdown() {
+            this.Logger.LogWarning("Shutdown Received.");
+
+            List<Task> tasks = [];
+
+            foreach (var id in this.IdsInProgress) {
+                tasks.Add(AzureStorage.DeleteFile($"{id}/generating"));
+            }
+
+            this.Logger.LogInformation("Ending {count} in-progress generations.", tasks.Count);
+
+            await Task.WhenAll(tasks);
+        }
+    }
+}