Stream Tar archives in PHP without memory limits

<?php
/**
 * Report whether an already-resolved path sits inside an already-resolved root.
 * Both arguments must come from realpath() so that symlinks and '..' are gone.
 * Passing '/' as a root allows the entire filesystem, so only configure it deliberately.
 */
function isWithin(string $path, string $root): bool
{
    return $path === $root || str_starts_with($path, rtrim($root, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR);
}

/**
 * Reduce a download name to a conservative ASCII attachment filename.
 * Everything outside [A-Za-z0-9._-] is replaced, so no quote, backslash, or control
 * character can escape the quoted-string in Content-Disposition. A backslash matters as
 * much as a quote here: "report\" ends the value one character later than it looks.
 */
function safeAttachmentName(string $downloadName, string $fallback = 'archive.tar'): string
{
    $name = trim((string) preg_replace('/[^A-Za-z0-9._-]+/', '_', basename($downloadName)), '._');

    return $name === '' ? $fallback : substr($name, 0, 100);
}

function clearOutputBuffers(): void
{
    // Any remaining framework buffer would accumulate the archive despite flush().
    while (ob_get_level() > 0) {
        $status = ob_get_status();
        if (($status['flags'] & PHP_OUTPUT_HANDLER_REMOVABLE) === 0 || !ob_end_clean()) {
            throw new RuntimeException('Cannot disable output buffering for this response.');
        }
    }
}

function streamTarArchive(string $directory, string $downloadName = 'archive.tar'): void
{
    $safeDownloadName = safeAttachmentName($downloadName);

    // tar reads a leading '-' as an option, so a directory literally named
    // "--checkpoint-action=exec=..." would run a command. realpath() gives an absolute
    // path, and '--' ends option parsing for anything it still cannot cover.
    $realDir = realpath($directory);
    if ($realDir === false || !is_dir($realDir)) {
        throw new InvalidArgumentException('Directory does not exist or is not accessible.');
    }

    clearOutputBuffers();
    $cmd = 'tar -C ' . escapeshellarg($realDir) . ' -cf - -- .';

    // stdin is never used by `tar -c`, and stderr is discarded. Capturing stderr would
    // mean either draining a second stream while stdout is mid-transfer, or buffering it,
    // and a chatty tar can emit one line per file. The exit code is the diagnostic we keep.
    $descriptorSpec = [
        0 => ['file', '/dev/null', 'r'],
        1 => ['pipe', 'w'],
        2 => ['file', '/dev/null', 'w'],
    ];
    $pipes = [];

    $process = proc_open($cmd, $descriptorSpec, $pipes);
    if (!is_resource($process)) {
        throw new RuntimeException('Failed to create tar process. Is tar installed and in PATH?');
    }

    header('Content-Type: application/x-tar');
    header('Content-Disposition: attachment; filename="' . $safeDownloadName . '"');
    header('X-Content-Type-Options: nosniff'); // Security: prevent MIME-sniffing
    header('Cache-Control: private, no-store'); // Build logs are confidential: do not cache

    // Stream the tar output. fread() returns '' at EOF and false on a real read error,
    // and those two have to stay distinguishable: treating false as EOF reports a
    // truncated archive as a complete one.
    $readFailed = false;
    while (true) {
        $chunk = fread($pipes[1], 8192); // Read in 8KB chunks
        if ($chunk === false) {
            $readFailed = true;
            break;
        }
        if ($chunk === '') {
            break; // EOF
        }
        echo $chunk;
        flush(); // Flush output to the client
    }

    fclose($pipes[1]);

    // Use one completion path: wait for tar and collect its exit code here.
    $exitCode = proc_close($process);

    if ($readFailed) {
        throw new RuntimeException('Failed to read the tar output stream.');
    }
    if ($exitCode !== 0) {
        throw new RuntimeException("tar exited with status {$exitCode}");
    }
}

<?php
// Mount this behind your existing auth middleware. It assumes the caller is already
// authenticated and already authorized for this project's build logs.
function downloadCiLogs(string $logDirIdentifier): void
{
    // Example: map an identifier to an actual path
    // In a real app, this might come from a config or database, scoped to the caller's project
    $logPathMappings = [
        'project-alpha-build-123' => '/var/logs/ci-cd/project-alpha/build-123',
        'project-beta-deploy-45' => '/var/logs/ci-cd/project-beta/deploy-45',
    ];

    if (!isset($logPathMappings[$logDirIdentifier])) {
        throw new InvalidArgumentException('Invalid log directory identifier.');
    }

    // Use realpath to resolve symbolic links and '..'
    $realDir = realpath($logPathMappings[$logDirIdentifier]);
    $allowedRoot = realpath('/var/logs/ci-cd'); // Ensure this base path is secure

    if ($realDir === false || !is_dir($realDir)) {
        throw new InvalidArgumentException('Directory does not exist or is not accessible.');
    }
    // Compare against the root plus a separator. A bare prefix check would also accept
    // a sibling such as /var/logs/ci-cd-evil for the root /var/logs/ci-cd.
    if ($allowedRoot === false || !isWithin($realDir, $allowedRoot)) {
        throw new RuntimeException('Access denied to the specified directory.');
    }

    streamTarArchive($realDir, 'ci-logs-' . $logDirIdentifier . '-' . date('Y-m-d') . '.tar');
}

// Example route handler, running after auth and authorization have already passed:
try {
    $logIdentifier = $_GET['log_id'] ?? '';
    if (!is_string($logIdentifier) || !preg_match('/^[a-zA-Z0-9_-]{1,64}$/', $logIdentifier)) {
        throw new InvalidArgumentException('Invalid log identifier format.');
    }
    downloadCiLogs($logIdentifier);
} catch (Throwable $e) {
    // Diagnostics stay server-side. Echoing $e->getMessage() would hand the client
    // filesystem paths and tar internals, and would confirm which identifiers exist.
    error_log('CI log download failed: ' . $e->getMessage());

    // headers_sent() is the whole story here: once streamTarArchive() has flushed a byte,
    // the 200 and its headers are already gone and nothing below can undo them.
    if (!headers_sent()) {
        header_remove();
        http_response_code($e instanceof InvalidArgumentException ? 400 : 500);
        header('Content-Type: text/plain; charset=utf-8');
        header('Cache-Control: private, no-store');
        echo "The archive could not be generated.\n";
    }
}

<?php
function sseTarProgress(string $directory, array $allowedRoots): void
{
    // Same authentication and authorization assumptions as downloadCiLogs() above.
    $realDir = realpath($directory);
    if ($realDir === false || !is_dir($realDir)) {
        throw new InvalidArgumentException('Directory does not exist or is not accessible.');
    }

    $isAllowed = false;
    foreach ($allowedRoots as $root) {
        $realRoot = realpath($root);
        if ($realRoot !== false && isWithin($realDir, $realRoot)) {
            $isAllowed = true;
            break;
        }
    }
    if (!$isAllowed) {
        throw new RuntimeException('Access denied to the specified directory for progress reporting.');
    }

    clearOutputBuffers();
    header('Content-Type: text/event-stream');
    header('Cache-Control: private, no-store');
    header('Connection: keep-alive'); // Important for SSE

    // -v sends one filename per file to stderr. The archive itself goes to /dev/null:
    // this run exists only to report progress, so stdout can never fill and block.
    $descriptorSpec = [
        0 => ['file', '/dev/null', 'r'], // stdin - not used by tar -c
        1 => ['file', '/dev/null', 'w'], // stdout - the archive, discarded
        2 => ['pipe', 'w'],              // stderr - where tar -v outputs filenames
    ];
    $pipes = [];

    $process = proc_open('tar -C ' . escapeshellarg($realDir) . ' -cvf - -- .', $descriptorSpec, $pipes);
    if (!is_resource($process)) {
        throw new RuntimeException('tar process failed to start for progress reporting.');
    }

    ignore_user_abort(true);
    set_time_limit(0); // Allow script to run indefinitely for long tar processes

    // A plain blocking read to EOF. Note what this does not do: there are no periodic
    // keep-alives, so a tar that stays silent for minutes sends nothing at all. If a proxy
    // between you and the client times idle connections out, drive this with
    // stream_select() on a timeout and emit a ": keepalive" comment when it expires.
    while (($line = fgets($pipes[2])) !== false) {
        $line = trim($line);
        if ($line === '') {
            continue;
        }
        echo 'data: ' . json_encode(['file' => $line]) . "\n\n";
        flush(); // Flush data to the client
    }
    fclose($pipes[2]);

    // Exactly one terminal event, chosen by tar's real exit status. Emitting "complete"
    // from a finally block would tell the client the archive is fine even when tar died.
    $exitCode = proc_close($process);
    if ($exitCode === 0) {
        echo "event: complete\n" . 'data: ' . json_encode(['done' => true]) . "\n\n";
    } else {
        error_log('sseTarProgress: tar exited with status ' . $exitCode);
        echo "event: error\n" . 'data: ' . json_encode(['message' => 'Archiving failed.']) . "\n\n";
    }
    flush();
}

<?php
class SecureTarStreamer
{
    private array $allowedRoots;

    public function __construct(array $allowedRoots)
    {
        // Ensure allowedRoots are absolute and valid paths during construction
        $this->allowedRoots = array_map(function($root) {
            $realRoot = realpath($root);
            if ($realRoot === false || !is_dir($realRoot)) {
                throw new InvalidArgumentException("Invalid allowed root directory: {$root}");
            }
            return $realRoot;
        }, $allowedRoots);
    }

    public function send(string $userSuppliedDir, string $downloadName = 'archive.tar'): void
    {
        $path = realpath($userSuppliedDir); // Resolve the user-supplied path

        if ($path === false || !is_dir($path)) {
            throw new InvalidArgumentException('Invalid or non-existent directory specified.');
        }

        $isAllowed = false;
        foreach ($this->allowedRoots as $root) {
            if (isWithin($path, $root)) {
                $isAllowed = true;
                break;
            }
        }

        if (!$isAllowed) {
            throw new RuntimeException('Access to the specified directory is not allowed.');
        }

        // Now it's safer to call the streaming function
        streamTarArchive($path, $downloadName);
    }
}

// Example Usage, again behind your existing authentication and authorization:
// $streamer = new SecureTarStreamer(['/var/www/safe_uploads', '/mnt/user_data']);
// $streamer->send($_GET['directory_to_archive']);
//
// The allowlist is checked with realpath(), so it inspects the tree and then hands the
// resolved path to tar. Between those two steps a symlink could be swapped. Keep the
// archived directories under your application's control rather than in a tree that other
// local users can write to.

{
  "steps": {
    "compressed": {
      "robot": "/file/compress",
      "use": ":original",
      "format": "tar",
      "gzip": true
    }
  }
}