Make darc add-subscription work with the new config repo

Directory.Packages.props

@@ -53,7 +53,6 @@
     <PackageVersion Include="Microsoft.DotNet.Internal.Testing.DependencyInjectionCodeGen" Version="$(MicrosoftDotNetInternalTestingDependencyInjectionCodeGenVersion)" />
     <PackageVersion Include="Microsoft.DotNet.Internal.Testing.Utility" Version="$(MicrosoftDotNetInternalTestingUtilityVersion)" />
     <PackageVersion Include="Microsoft.DotNet.Kusto" Version="$(MicrosoftDotNetKustoVersion)" />
-    <PackageVersion Include="Microsoft.DotNet.MaestroConfiguration.Client" Version="$(MicrosoftDotNetMaestroConfigurationClientVersion)" />
     <PackageVersion Include="Microsoft.DotNet.Services.Utility" Version="$(MicrosoftDotNetServicesUtilityVersion)" />
     <PackageVersion Include="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="$(MicrosoftDotNetSwaggerGeneratorMSBuildVersion)" />
     <PackageVersion Include="Microsoft.DotNet.VersionTools" Version="$(MicrosoftDotNetVersionToolsVersion)" />
@@ -107,5 +106,8 @@
     <!-- Pinned to clear a CG alert (this package comes via Microsoft.DotNet.Kusto but didn't get updated there yet) -->
     <PackageVersion Include="Microsoft.Azure.Kusto.Data" Version="14.0.3" />
     <PackageVersion Include="Microsoft.Azure.Kusto.Ingest" Version="14.0.3" />
+    <!-- remove once the MaestrConfiguration.Client has been moved back to it's own repo -->
+    <PackageVersion Include="xunit" Version="2.9.3" />
+    <PackageVersion Include="coverlet.collector" Version="6.0.4" />
   </ItemGroup>
 </Project>

arcade-services.slnx

@@ -4,6 +4,10 @@
     <Platform Name="x64" />
     <Platform Name="x86" />
   </Configurations>
+  <Folder Name="/MaestroConfiguration/">
+    <Project Path="src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/Microsoft.DotNet.MaestroConfiguration.Client.csproj" />
+    <Project Path="src/MaestroConfiguration/test/Microsoft.DotNet.MaestroConfiguration.Client.Tests/Microsoft.DotNet.MaestroConfiguration.Client.Tests.csproj" />
+  </Folder>
   <Folder Name="/Solution Items/">
     <File Path=".editorconfig" />
     <File Path="azure-pipelines.yml" />

eng/Version.Details.props

@@ -28,8 +28,6 @@ This file should be imported by eng/Versions.props
     <!-- dotnet/dnceng dependencies -->
     <MicrosoftDncEngConfigurationBootstrapPackageVersion>1.1.0-beta.25608.3</MicrosoftDncEngConfigurationBootstrapPackageVersion>
     <MicrosoftDncEngSecretManagerPackageVersion>1.1.0-beta.25608.3</MicrosoftDncEngSecretManagerPackageVersion>
-    <!-- _git/maestro-configuration dependencies -->
-    <MicrosoftDotNetMaestroConfigurationClientPackageVersion>0.0.99-beta.25604.1</MicrosoftDotNetMaestroConfigurationClientPackageVersion>
   </PropertyGroup>
   <!--Property group for alternate package version names-->
   <PropertyGroup>
@@ -56,7 +54,5 @@ This file should be imported by eng/Versions.props
     <!-- dotnet/dnceng dependencies -->
     <MicrosoftDncEngConfigurationBootstrapVersion>$(MicrosoftDncEngConfigurationBootstrapPackageVersion)</MicrosoftDncEngConfigurationBootstrapVersion>
     <MicrosoftDncEngSecretManagerVersion>$(MicrosoftDncEngSecretManagerPackageVersion)</MicrosoftDncEngSecretManagerVersion>
-    <!-- _git/maestro-configuration dependencies -->
-    <MicrosoftDotNetMaestroConfigurationClientVersion>$(MicrosoftDotNetMaestroConfigurationClientPackageVersion)</MicrosoftDotNetMaestroConfigurationClientVersion>
   </PropertyGroup>
 </Project>

eng/Version.Details.xml

@@ -83,9 +83,5 @@
       <Uri>https://github.com/dotnet/dnceng</Uri>
       <Sha>6b49135c494c814a01232e423244f14241cf09b4</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.MaestroConfiguration.Client" Version="0.0.99-beta.25604.1">
-      <Uri>https://dev.azure.com/dnceng/internal/_git/maestro-configuration</Uri>
-      <Sha>94627a0f38e4bae7e7276bb2155dac4a66de0a79</Sha>
-    </Dependency>
   </ToolsetDependencies>
 </Dependencies>

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/ConfigFilePathResolver.cs

@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Linq;
+using Maestro.Common;
+using Microsoft.DotNet.MaestroConfiguration.Client.Models;
+using Microsoft.DotNet.ProductConstructionService.Client.Helpers;
+using Microsoft.DotNet.ProductConstructionService.Client.Models;
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public static class ConfigFilePathResolver
+{
+    private static string ConfigurationFolderPath = new("configuration");
+    private const string SubscriptionFolder = "subscriptions";
+    private const string ChannelFolder = "channels";
+    private const string DefaultChannelFolder = "default-channels";
+    private const string RepositoryBranchFolder = "branch-merge-policies";
+    private const string YamlFileExtension = ".yml";
+
+    public static string SubscriptionFolderPath => Path.Combine(ConfigurationFolderPath, SubscriptionFolder);
+    public static string ChannelFolderPath => Path.Combine(ConfigurationFolderPath, ChannelFolder);
+    public static string DefaultChannelFolderPath => Path.Combine(ConfigurationFolderPath, DefaultChannelFolder);
+    public static string RepositoryBranchFolderPath => Path.Combine(ConfigurationFolderPath, RepositoryBranchFolder);
+
+    public static string GetDefaultSubscriptionFilePath(SubscriptionYaml subscription) =>
+        Path.Combine(SubscriptionFolderPath, GetFileNameBasedOnRepo(subscription.TargetRepository));
+
+    public static string GetDefaultChannelFilePath(ChannelYaml channel) =>
+        NormalizeChannelName(Path.Combine(ChannelFolderPath, (ChannelCategorizer.CategorizeChannels([new Channel(0, channel.Name, string.Empty)]).First().Name + YamlFileExtension)));
+
+    public static string GetDefaultDefaultChannelFilePath(DefaultChannelYaml defaultChannel) =>
+        Path.Combine(DefaultChannelFolderPath, GetFileNameBasedOnRepo(defaultChannel.Repository));
+
+    public static string GetDefaultRepositoryBranchFilePath(BranchMergePoliciesYaml branchMergePolicies) =>
+        Path.Combine(RepositoryBranchFolderPath, GetFileNameBasedOnRepo(branchMergePolicies.Repository));
+
+    private static string GetFileNameBasedOnRepo(string repository)
+    {
+        try
+        {
+            var (repoName, owner) = GitRepoUrlUtils.GetRepoNameAndOwner(repository);
+            return $"{owner}-{repoName}{YamlFileExtension}";
+        }
+        catch (ArgumentException)
+        {
+            if (GitRepoUrlUtils.ParseTypeFromUri(repository) == GitRepoType.AzureDevOps)
+            {
+                return repository.Split('/', StringSplitOptions.RemoveEmptyEntries).Last() + YamlFileExtension;
+            }
+            else
+            {
+                throw;
+            }
+        }
+    }
+
+    private static string NormalizeChannelName(string channelName) =>
+        channelName.ToLowerInvariant().Replace(" ", "-");
+}

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/ConfigurationRepositoryManager.cs

@@ -0,0 +1,194 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.DotNet.MaestroConfiguration.Client.Models;
+using Microsoft.Extensions.Logging;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public class ConfigurationRepositoryManager : IConfigurationRepositoryManager
+{
+    private readonly IGitRepoFactory _configurationRepoFactory;
+    private readonly ILogger<IConfigurationRepositoryManager> _logger;
+
+    private static readonly ISerializer _yamlSerializer = new SerializerBuilder()
+        .WithNamingConvention(NullNamingConvention.Instance)
+        .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults | DefaultValuesHandling.OmitEmptyCollections)
+        .Build();
+
+    private static readonly IDeserializer _yamlDeserializer = new DeserializerBuilder()
+        .WithNamingConvention(NullNamingConvention.Instance)
+        .Build();
+
+    public ConfigurationRepositoryManager(
+        IGitRepoFactory configurationRepoFactory,
+        ILogger<IConfigurationRepositoryManager> logger)
+    {
+        _logger = logger;
+        _configurationRepoFactory = configurationRepoFactory;
+    }
+
+    public async Task AddSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, SubscriptionYaml subscription)
+    {
+        IGitRepo configurationRepo = await _configurationRepoFactory.CreateClient(parameters.RepositoryUri);
+
+        await ValidateConfigurationRepositoryParametersAsync(configurationRepo, parameters);
+        var workingBranch = await PrepareConfigurationBranchAsync(configurationRepo, parameters);
+
+        var newSubscriptionFilePath = string.IsNullOrEmpty(parameters.ConfigurationFilePath)
+            ? ConfigFilePathResolver.GetDefaultSubscriptionFilePath(subscription)
+            : parameters.ConfigurationFilePath;
+        _logger.LogInformation("Adding new subscription to file {0}", newSubscriptionFilePath);
+
+        var subscriptionsInFile = await FetchAndParseRemoteConfiguration<SubscriptionYaml>(
+            configurationRepo,
+            parameters.RepositoryUri,
+            workingBranch,
+            newSubscriptionFilePath);
+
+        // If we have a branch that hasn't been ingested yet, we need to check for equivalent subscriptions in the file
+        var equivalentInFile = subscriptionsInFile.FirstOrDefault(s => s.IsEquivalentTo(subscription));
+        if (equivalentInFile != null)
+        {
+            throw new ArgumentException($"Subscription {equivalentInFile.Id} with equivalent parameters already exists in '{newSubscriptionFilePath}'.");
+        }
+
+        subscriptionsInFile.Add(subscription);
+        await CommitConfigurationDataAsync(
+            configurationRepo,
+            parameters.RepositoryUri,
+            workingBranch,
+            newSubscriptionFilePath,
+            subscriptionsInFile,
+            $"Add new subscription ({subscription.Channel}) {subscription.SourceRepository} => {subscription.TargetRepository} ({subscription.TargetBranch})");
+
+        if (!parameters.DontOpenPr)
+        {
+            // Open a pull request for the new subscription
+            await CreatePullRequest(
+                configurationRepo,
+                parameters.RepositoryUri,
+                workingBranch,
+                parameters.ConfigurationBaseBranch,
+                newSubscriptionFilePath,
+                "Updating Maestro configuration");
+        }
+        else
+        {
+            _logger.LogInformation("Successfully added subscription with id '{0}' to branch '{1}' of the configuration repository {2}",
+                subscription.Id, parameters.ConfigurationBranch, parameters.RepositoryUri);
+        }
+    }
+
+    private static async Task ValidateConfigurationRepositoryParametersAsync(
+        IGitRepo gitRepo,
+        ConfigurationRepositoryOperationParameters operationParameters)
+    {
+        if (!await gitRepo.RepoExistsAsync(operationParameters.RepositoryUri))
+        {
+            throw new ArgumentException($"The configuration repository '{operationParameters.RepositoryUri}' is not a valid git repository.");
+        }
+
+        if (!await gitRepo.DoesBranchExistAsync(operationParameters.RepositoryUri, operationParameters.ConfigurationBaseBranch))
+        {
+            throw new ArgumentException($"The configuration base branch '{operationParameters.ConfigurationBaseBranch}' does not exist in the repository '{operationParameters.RepositoryUri}'.");
+        }
+    }
+
+    private static async Task CommitConfigurationDataAsync<T>(
+        IGitRepo gitRepo,
+        string repositoryUri,
+        string workingBranch,
+        string filePath,
+        IEnumerable<T> data,
+        string commitMessage)
+        where T : IYamlModel
+    {
+        string yamlContent = _yamlSerializer.Serialize(YamlModelSorter.Sort(data)).Replace("\n-", "\n\n-");
+        await gitRepo.CommitFilesAsync(repositoryUri, workingBranch, [new GitFile(filePath, yamlContent)], commitMessage);
+    }
+
+    /// <summary>
+    /// Ensures that a configuration working branch exists, creating one if necessary.
+    /// </summary>
+    private static async Task<string> PrepareConfigurationBranchAsync(
+        IGitRepo gitRepo,
+        ConfigurationRepositoryOperationParameters parameters)
+    {
+        if (string.IsNullOrEmpty(parameters.ConfigurationBranch))
+        {
+            var branch = $"darc/{parameters.ConfigurationBaseBranch}-{Guid.NewGuid().ToString().Substring(0, 8)}";
+            await gitRepo.CreateBranchAsync(
+                parameters.RepositoryUri,
+                branch,
+                parameters.ConfigurationBaseBranch);
+            return branch;
+        }
+        else
+        {
+            if (!await gitRepo.DoesBranchExistAsync(parameters.RepositoryUri, parameters.ConfigurationBranch))
+            {
+                await gitRepo.CreateBranchAsync(
+                    parameters.RepositoryUri,
+                    parameters.ConfigurationBranch,
+                    parameters.ConfigurationBaseBranch);
+            }
+            return parameters.ConfigurationBranch;
+        }
+    }
+
+    private static async Task<List<TData>> FetchAndParseRemoteConfiguration<TData>(
+        IGitRepo gitRepo,
+        string repositoryUri,
+        string workingBranch,
+        string filePath)
+    {
+        string fileContents;
+
+        try
+        {
+            fileContents = await gitRepo.GetFileContentsAsync(
+                repositoryUri,
+                workingBranch,
+                filePath);
+            return _yamlDeserializer.Deserialize<List<TData>>(fileContents);
+        }
+        catch (FileNotFoundInRepoException)
+        {
+            return [];
+        }   
+    }
+
+    private async Task CreatePullRequest(
+        IGitRepo gitRepo,
+        string repositoryUri,
+        string headBranch,
+        string targetBranch,
+        string title,
+        string? description = null)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(title);
+        ArgumentException.ThrowIfNullOrEmpty(headBranch);
+        ArgumentException.ThrowIfNullOrEmpty(targetBranch);
+
+        _logger.LogInformation("Creating pull request from {0} to {1}...", headBranch, targetBranch);
+        var prUrl = await gitRepo.CreatePullRequestAsync(
+            repositoryUri,
+            headBranch,
+            targetBranch,
+            title,
+            description);
+        var prId = prUrl.Substring(prUrl.LastIndexOf('/') + 1);
+        var guiUri = $"{repositoryUri}/pullrequest/{prId}";
+        _logger.LogInformation("Created pull request at {0}", guiUri);
+    }
+
+    public Task DeleteSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, Guid subscriptionId) => throw new NotImplementedException();
+    public Task UpdateSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, SubscriptionYaml updatedSubscription) => throw new NotImplementedException();
+}

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/ConfigurationRepositoryOperationParameters.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public class ConfigurationRepositoryOperationParameters
+{
+    public required string RepositoryUri { get; init; }
+    public required string ConfigurationBaseBranch { get; init; }
+    public string? ConfigurationBranch { get; set; }
+    public required bool DontOpenPr { get; init; }
+    public string? ConfigurationFilePath { get; init; }
+}

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/FileNotFoundInRepoException.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public class FileNotFoundInRepoException : Exception
+{
+    public FileNotFoundInRepoException(string repositoryUri, string branchName, string filePath)
+        : base($"The file '{filePath}' was not found in repository '{repositoryUri}' on branch '{branchName}'.")
+    {
+    }
+}

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/GitFile.cs

@@ -0,0 +1,6 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public record GitFile(string Path, string Content);

src/MaestroConfiguration/src/Microsoft.DotNet.MaestroConfiguration.Client/IConfigurationRepositoryManager.cs

@@ -0,0 +1,15 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.DotNet.MaestroConfiguration.Client.Models;
+
+namespace Microsoft.DotNet.MaestroConfiguration.Client;
+
+public interface IConfigurationRepositoryManager
+{
+    Task AddSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, SubscriptionYaml subscription);
+    Task DeleteSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, Guid subscriptionId);
+    Task UpdateSubscriptionAsync(ConfigurationRepositoryOperationParameters parameters, SubscriptionYaml updatedSubscription);
+}