Import files from Cloudflare R2 in Java with Rclone

# Install Rclone (linux/macos/bsd)
curl https://rclone.org/install.sh | sudo bash

# Verify installation
rclone version

# Configure Rclone (interactive)
rclone config

[cloudflare_r2]
type = s3
provider = Cloudflare
access_key_id = YOUR_ACCESS_KEY_ID
secret_access_key = YOUR_SECRET_ACCESS_KEY
endpoint = https://<accountid>.r2.cloudflarestorage.com
acl = private

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit; // For timeout handling

public class RcloneImporter {

    /**
     * Imports files from a specified Cloudflare R2 path to a local path using Rclone.
     *
     * @param remoteName The name of the configured Rclone remote (e.g., "cloudflare_r2").
     * @param remotePath The path within the R2 bucket (e.g., "my-bucket/path/to/files").
     * @param localPath  The local directory path where files will be downloaded.
     * @throws IOException          If an I/O error occurs during process execution.
     * @throws InterruptedException If the current thread is interrupted while waiting for the process.
     * @throws RuntimeException     If the Rclone command fails (non-zero exit code) or times out.
     */
    public static void importFiles(String remoteName, String remotePath, String localPath)
            throws IOException, InterruptedException {
        List<String> command = new ArrayList<>();
        command.add("rclone");
        command.add("copy"); // Use "sync" for synchronization instead of just copying
        command.add(remoteName + ":" + remotePath); // Format: remote:path/to/dir
        command.add(localPath); // Destination local directory

        // Example: Add flags for parallel transfers and progress
        // command.add("--transfers");
        // command.add("8");
        // command.add("--progress");

        System.out.println("Executing Rclone command: " + String.join(" ", command));

        ProcessBuilder builder = new ProcessBuilder(command);
        builder.redirectErrorStream(true); // Merge error stream with standard output

        Process process = builder.start();

        // Capture and print process output
        try (BufferedReader reader = new BufferedReader(
                new InputStreamReader(process.getInputStream()))) {
            String line;
            while ((line = reader.readLine()) != null) {
                // Replace with proper logging in a real application
                System.out.println("Rclone Output: " + line);
            }
        }

        // Wait for the process to complete with a timeout
        // Adjust timeout value as needed
        boolean finished = process.waitFor(10, TimeUnit.MINUTES);

        if (!finished) {
            process.destroyForcibly();
            throw new RuntimeException("Rclone command timed out after 10 minutes.");
        }

        int exitCode = process.exitValue(); // Use exitValue() after waitFor()
        if (exitCode != 0) {
            // Consider more specific exception handling based on Rclone exit codes if needed
            throw new RuntimeException("Rclone command failed with exit code: " + exitCode);
        } else {
            System.out.println("Rclone command executed successfully.");
        }
    }

    public static void main(String[] args) {
        // Example usage:
        String rcloneRemoteName = "cloudflare_r2"; // Matches the name used in `rclone config`
        String bucketPath = "my-data-bucket/source-files"; // Path inside your R2 bucket
        String localDirectory = "./downloaded-files"; // Local destination directory

        try {
            // Optional: Ensure the local directory exists
            // java.nio.file.Files.createDirectories(java.nio.file.Paths.get(localDirectory));

            System.out.println("Starting file import from Cloudflare R2...");
            importFiles(rcloneRemoteName, bucketPath, localDirectory);
            System.out.println("File import completed successfully.");

        } catch (IOException | InterruptedException e) {
            System.err.println("Error during Rclone execution: " + e.getMessage());
            // Log the exception stack trace for debugging
            e.printStackTrace();
            // Handle the error appropriately in your application
            Thread.currentThread().interrupt(); // Restore interrupted status
        } catch (RuntimeException e) {
            System.err.println("Rclone command failed or timed out: " + e.getMessage());
            // Log the exception stack trace for debugging
            e.printStackTrace();
            // Handle the error appropriately in your application
        }
    }

    // --- Advanced Usage Examples ---

    /**
     * Lists object names (files and directories) in a given remote path.
     */
    public static List<String> listObjects(String remoteName, String remotePath)
            throws IOException, InterruptedException {
        List<String> command = List.of("rclone", "lsf", remoteName + ":" + remotePath);
        ProcessBuilder builder = new ProcessBuilder(command);
        builder.redirectErrorStream(true);
        Process process = builder.start();

        List<String> objectNames = new ArrayList<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
            String line;
            while ((line = reader.readLine()) != null) {
                objectNames.add(line.trim());
            }
        }

        boolean finished = process.waitFor(1, TimeUnit.MINUTES); // Shorter timeout for listing
        if (!finished) {
            process.destroyForcibly();
            throw new RuntimeException("Rclone list command timed out.");
        }

        int exitCode = process.exitValue();
        if (exitCode != 0) {
            // Rclone might return non-zero if path doesn't exist, handle appropriately
            System.err.println("Rclone list command finished with non-zero exit code: " + exitCode);
            // Depending on the use case, you might return an empty list or throw
            // throw new RuntimeException("Rclone list command failed with exit code: " + exitCode);
        }
        return objectNames;
    }

    /**
     * Checks if a specific file exists at the given remote path.
     */
    public static boolean fileExists(String remoteName, String remoteFilePath)
            throws IOException, InterruptedException {
        // Using `lsf` on a specific file path is a common way to check existence
        List<String> command = List.of("rclone", "lsf", remoteName + ":" + remoteFilePath);
        ProcessBuilder builder = new ProcessBuilder(command);
        builder.redirectErrorStream(true);
        Process process = builder.start();

        StringBuilder output = new StringBuilder();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
            String line;
            // Read the first line only, if it exists, the file is there
            if ((line = reader.readLine()) != null) {
                 output.append(line);
            }
            // Consume rest of output if any
            while (reader.readLine() != null) {}
        }

        boolean finished = process.waitFor(30, TimeUnit.SECONDS); // Timeout for check
        if (!finished) {
            process.destroyForcibly();
            throw new RuntimeException("Rclone file check command timed out.");
        }

        int exitCode = process.exitValue();
        // Rclone lsf returns exit code 0 and the filename if found.
        // Non-zero exit code (e.g., 3 for "Directory not found") indicates not found or error.
        return exitCode == 0 && !output.toString().trim().isEmpty();
    }
}

// Caution: Running 'rclone config create' programmatically can be complex
// and might expose secrets in process lists or logs if not handled carefully.
// Prefer configuring Rclone beforehand using its config file or standard env vars.
public static void configureRcloneFromEnv() throws IOException, InterruptedException {
    String remoteName = "cloudflare_r2_env"; // Use a distinct name
    String accessKey = System.getenv("R2_ACCESS_KEY_ID");
    String secretKey = System.getenv("R2_SECRET_KEY"); // Use secure env var names
    String endpoint = System.getenv("R2_ENDPOINT");

    if (accessKey == null || secretKey == null || endpoint == null) {
        throw new IllegalStateException("Required R2 environment variables are not set.");
    }

    // Note: Passing secrets directly on the command line is generally discouraged.
    // Rclone has safer ways (env vars like RCLONE_CONFIG_REMOTE_ACCESS_KEY_ID).
    // This example is illustrative; review Rclone docs for secure credential handling.
    List<String> command = List.of(
        "rclone", "config", "create", remoteName, "s3",
        "provider=Cloudflare",
        "access_key_id=" + accessKey,
        "secret_access_key=" + secretKey, // HIGHLY CAUTIOUS with this approach
        "endpoint=" + endpoint,
        "acl=private"
        // Consider using "config_is_local=true" if needed
    );

    System.out.println("Attempting to configure Rclone dynamically (use with extreme caution)...");
    ProcessBuilder builder = new ProcessBuilder(command);
    builder.redirectErrorStream(true);
    Process process = builder.start();

    // Capture output for debugging (avoid in production if secrets are involved)
    try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
        String line;
        while ((line = reader.readLine()) != null) {
            System.out.println("Config Output: " + line);
        }
    }

    boolean finished = process.waitFor(1, TimeUnit.MINUTES);
    if (!finished) {
        process.destroyForcibly();
        throw new RuntimeException("Failed to configure Rclone dynamically: Timeout");
    }

    int exitCode = process.exitValue();
    if (exitCode != 0) {
        throw new RuntimeException("Failed to configure Rclone dynamically. Exit code: " + exitCode);
    }
    System.out.println("Rclone remote '" + remoteName + "' configured dynamically (verify security implications).");
}