Exporter facilement des fichiers vers OpenStack Swift en .NET C#

if ! command -v dotnet >/dev/null 2>&1; then
  printf '%s\n' 'Install the .NET 10 SDK first.' >&2
  false
elif [ -e SwiftExport ] || [ -L SwiftExport ]; then
  printf '%s\n' 'SwiftExport already exists; choose another parent directory.' >&2
  false
else
  dotnet new console --framework net10.0 --name SwiftExport --no-restore &&
  dotnet restore SwiftExport/SwiftExport.csproj &&
  cd SwiftExport
fi

using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text.Json;

public static class Program
{
    public static async Task<int> Main(string[] args)
    {
        using var deadline = new CancellationTokenSource(TimeSpan.FromMinutes(5));
        Console.CancelKeyPress += (_, e) => { e.Cancel = true; deadline.Cancel(); };
        try
        {
            if (args.Length != 3) throw new ArgumentException("Pass container, object key, and local file.");
            if (args[2].Any(character => character > 127) ||
                Directory.GetCurrentDirectory().Any(character => character > 127))
                throw new ArgumentException("Use ASCII local paths and working directory names.");
            var endpoint = new Uri(Required("SWIFT_STORAGE_URL"));
            if (endpoint.Scheme != "https" || endpoint.UserInfo.Length != 0 ||
                endpoint.Query.Length != 0 || endpoint.Fragment.Length != 0)
                throw new ArgumentException("Use the HTTPS account endpoint from the service catalog.");
            using var handler = new SocketsHttpHandler { AllowAutoRedirect = false, MaxConnectionsPerServer = 4 };
            using var http = new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan };
            var exporter = new SwiftExporter(http, endpoint, Required("SWIFT_TOKEN"));
            await exporter.UploadAsync(args[0], args[1], args[2], deadline.Token);
            Console.WriteLine("Uploaded.");
            return 0;
        }
        catch (OperationCanceledException) { Console.Error.WriteLine("Upload canceled or timed out."); }
        catch (HttpRequestException error) { Console.Error.WriteLine($"Swift request failed (HTTP {error.StatusCode})."); }
        catch (Exception) { Console.Error.WriteLine("Upload failed. Check the file and Swift configuration."); }
        return 1;
    }

    private static string Required(string name) =>
        Environment.GetEnvironmentVariable(name) is { Length: > 0 } value
            ? value : throw new ArgumentException($"Missing {name}.");
}

public sealed class SwiftExporter(HttpClient http, Uri endpoint, string token)
{
    private const int SegmentSize = 5 * 1024 * 1024;
    private const int MaxSegments = 1000;

    private static string Encode(string value)
    {
        var parts = value.Split('/');
        if (parts.Any(part => part.Length == 0 || part == "." || part == ".."))
            throw new ArgumentException("Use nonempty path segments other than dot or dot-dot.");
        return string.Join("/", parts.Select(Uri.EscapeDataString));
    }

    private async Task<string?> PutAsync(string path, HttpContent content,
        bool container, CancellationToken ct, string? etag = null, string query = "")
    {
        content.Headers.ContentType ??= new MediaTypeHeaderValue("application/octet-stream");
        using var request = new HttpRequestMessage(HttpMethod.Put,
            endpoint.AbsoluteUri.TrimEnd('/') + "/" + Encode(path) + query) { Content = content };
        request.Headers.Add("X-Auth-Token", token);
        if (etag != null) request.Headers.Add("ETag", etag);
        using var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct);
        var status = (int)response.StatusCode;
        if (status != 201 && !(container && status == 202))
            throw new HttpRequestException("Unexpected Swift response.", null, response.StatusCode);
        // Swift can return an unquoted MD5 that the typed ETag accessor cannot parse.
        return response.Headers.TryGetValues("ETag", out var values)
            ? values.Single().Trim('"') : null;
    }

    public async Task UploadAsync(string container, string key, string file, CancellationToken ct)
    {
        if (container.Contains('/')) throw new ArgumentException("A container cannot contain slashes.");
        Encode(container);
        Encode(key);
        await using var input = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
        long length = input.Length;
        if (length > (long)SegmentSize * MaxSegments) throw new IOException("File exceeds the example limit.");
        await PutAsync(container, new ByteArrayContent([]), true, ct);
        if (length == 0)
        {
            await PutAsync(container + "/" + key, new ByteArrayContent([]), false, ct);
            return;
        }

        string segments = container + "_segments";
        await PutAsync(segments, new ByteArrayContent([]), true, ct);
        string version = Guid.NewGuid().ToString("N");
        var manifest = new List<object>();
        var attempted = new List<string>();
        bool publicationStarted = false;
        var buffer = new byte[SegmentSize];
        long remaining = length;
        try
        {
            for (int index = 0; remaining > 0; index++)
            {
                int count = (int)Math.Min(remaining, SegmentSize);
                await input.ReadExactlyAsync(buffer.AsMemory(0, count), ct);
                string hash = Convert.ToHexString(MD5.HashData(buffer.AsSpan(0, count))).ToLowerInvariant();
                string name = $"{segments}/{version}/{index:D8}";
                attempted.Add(name);
                using var content = new ByteArrayContent(buffer, 0, count);
                string? returned = await PutAsync(name, content, false, ct, hash);
                if (!string.Equals(hash, returned, StringComparison.OrdinalIgnoreCase))
                    throw new IOException("Swift did not confirm the segment checksum.");
                manifest.Add(new { path = "/" + name, etag = hash, size_bytes = count });
                remaining -= count;
            }
            if (input.Length != length || input.ReadByte() != -1) throw new IOException("The input file changed.");
            using var manifestContent = new StringContent(JsonSerializer.Serialize(manifest));
            manifestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
            // After dispatch, a lost response cannot tell us whether these segments are live.
            publicationStarted = true;
            await PutAsync(container + "/" + key, manifestContent, false, ct,
                query: "?multipart-manifest=put");
        }
        catch
        {
            if (!publicationStarted) await RemoveAttemptedAsync(attempted);
            throw;
        }
    }

    private async Task RemoveAttemptedAsync(List<string> paths)
    {
        using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(15));
        foreach (string path in paths)
        {
            try
            {
                using var request = new HttpRequestMessage(HttpMethod.Delete,
                    endpoint.AbsoluteUri.TrimEnd('/') + "/" + Encode(path));
                request.Headers.Add("X-Auth-Token", token);
                using var response = await http.SendAsync(request,
                    HttpCompletionOption.ResponseHeadersRead, cleanup.Token);
                if ((int)response.StatusCode != 204 && (int)response.StatusCode != 404)
                    Console.Error.WriteLine("An unreferenced segment could not be removed.");
            }
            catch (HttpRequestException) { Console.Error.WriteLine("Segment cleanup lost its connection."); }
            catch (OperationCanceledException) { Console.Error.WriteLine("Segment cleanup timed out."); break; }
        }
    }
}

dotnet run -- exports 'reports/report #1.bin' report.bin