[CmdletBinding(SupportsShouldProcess)] param( [string]$SshTarget = 'root@akko.pchuan.top', [string]$RemoteRoot = '/root/cpa', [string]$WslDistribution = 'Debian', [int]$DrainTimeoutSeconds = 120, [int]$DrainPollSeconds = 5, [string]$BinaryPath = '', [switch]$SkipBuild, [switch]$SkipTests, [switch]$SkipUpstreamCheck ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest function Resolve-RequiredTool { param([Parameter(Mandatory)][string]$Name) $command = Get-Command $Name -ErrorAction SilentlyContinue if ($null -eq $command) { throw "Required tool is missing from PATH: $Name" } return $command.Source } function Invoke-NativeCommand { param( [Parameter(Mandatory)][string]$FilePath, [Parameter(Mandatory)][string[]]$ArgumentList ) & $FilePath @ArgumentList if ($LASTEXITCODE -ne 0) { throw "Command failed with exit code $LASTEXITCODE`: $FilePath $($ArgumentList -join ' ')" } } function Get-NativeOutput { param( [Parameter(Mandatory)][string]$FilePath, [Parameter(Mandatory)][string[]]$ArgumentList ) $output = & $FilePath @ArgumentList 2>&1 if ($LASTEXITCODE -ne 0) { throw "Command failed with exit code $LASTEXITCODE`: $FilePath $($ArgumentList -join ' ')`n$($output -join "`n")" } return (($output -join "`n").Trim()) } function Write-Utf8NoBom { param( [Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][string]$Content ) $encoding = [System.Text.UTF8Encoding]::new($false) [System.IO.File]::WriteAllText($Path, ($Content -replace "`r`n", "`n"), $encoding) } if ($DrainTimeoutSeconds -lt 0) { throw 'DrainTimeoutSeconds must be zero or greater.' } if ($DrainPollSeconds -lt 1) { throw 'DrainPollSeconds must be at least one second.' } if ($SshTarget -notmatch '^[A-Za-z0-9_.@-]+$') { throw "Unsupported SSH target format: $SshTarget" } if ($RemoteRoot -notmatch '^/[A-Za-z0-9._/-]+$') { throw "Unsupported remote root format: $RemoteRoot" } if ($WslDistribution -notmatch '^[A-Za-z0-9._-]+$') { throw "Unsupported WSL distribution name: $WslDistribution" } $git = Resolve-RequiredTool 'git' $ssh = Resolve-RequiredTool 'ssh' $scp = Resolve-RequiredTool 'scp' $wsl = Resolve-RequiredTool 'wsl.exe' $repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path $hostRoot = Join-Path $repoRoot '.externals\CLIProxyAPI' $runtimeRoot = Join-Path $repoRoot '.runtime' $tmpRoot = Join-Path $runtimeRoot 'tmp' $deployRoot = Join-Path $runtimeRoot 'deploy' if (-not (Test-Path -LiteralPath $hostRoot -PathType Container)) { throw "CLIProxyAPI submodule is missing: $hostRoot" } [System.IO.Directory]::CreateDirectory($tmpRoot) | Out-Null [System.IO.Directory]::CreateDirectory($deployRoot) | Out-Null Push-Location $repoRoot try { if (-not $SkipUpstreamCheck) { Invoke-NativeCommand $git @('-C', $hostRoot, 'fetch', '--tags', '--prune', 'origin') } $head = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', 'HEAD') if (-not $SkipUpstreamCheck) { $originMain = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', 'origin/main') if ($head -ne $originMain) { throw "Submodule HEAD is not origin/main. HEAD=$head origin/main=$originMain" } } $tag = Get-NativeOutput $git @('-C', $hostRoot, 'describe', '--tags', '--abbrev=0', $head) $commit = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', '--short=8', $head) $version = "$tag-dev" $buildDate = [DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ') foreach ($value in @($tag, $version, $commit)) { if ($value -notmatch '^[A-Za-z0-9._+-]+$') { throw "Unsupported version value: $value" } } Invoke-NativeCommand $git @('-C', $hostRoot, 'diff', '--check') $expectedModifiedFiles = Get-ChildItem -LiteralPath (Join-Path $repoRoot 'patch') -Filter '*.patch' | Select-String -Pattern '^diff --git a/([^ ]+) b/' | ForEach-Object { $_.Matches[0].Groups[1].Value } | Sort-Object -Unique $actualModifiedFiles = Get-NativeOutput $git @('-C', $hostRoot, 'diff', '--name-only') | ForEach-Object { $_ -split "`n" } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim() } | Sort-Object -Unique $untrackedHostFiles = Get-NativeOutput $git @('-C', $hostRoot, 'ls-files', '--others', '--exclude-standard') if (-not [string]::IsNullOrWhiteSpace($untrackedHostFiles)) { throw "CLIProxyAPI worktree contains untracked files:`n$untrackedHostFiles" } $unexpectedChanges = Compare-Object -ReferenceObject $expectedModifiedFiles -DifferenceObject $actualModifiedFiles if ($unexpectedChanges) { $details = $unexpectedChanges | ForEach-Object { "$($_.SideIndicator) $($_.InputObject)" } throw "CLIProxyAPI worktree does not match the files declared by patch/*.patch:`n$($details -join "`n")" } $wslRepoRoot = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($repoRoot -replace '\\', '/')) $archiveName = "cli-proxy-api-$version-$commit-linux-amd64.tar.gz" $archivePath = Join-Path $deployRoot $archiveName $checksumPath = "$archivePath.sha256" $wslArchivePath = "$wslRepoRoot/.runtime/deploy/$archiveName" if ($SkipBuild) { if ([string]::IsNullOrWhiteSpace($BinaryPath)) { $BinaryPath = Join-Path $runtimeRoot 'bin\cli-proxy-api' } $resolvedBinary = (Resolve-Path -LiteralPath $BinaryPath).Path $wslBinaryPath = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($resolvedBinary -replace '\\', '/')) $buildMode = '0' } else { $resolvedBinary = Join-Path $deployRoot "cli-proxy-api-$version-$commit" $wslBinaryPath = "$wslRepoRoot/.runtime/deploy/cli-proxy-api-$version-$commit" $buildMode = '1' } $runTests = if ($SkipTests) { '0' } else { '1' } $localScriptPath = Join-Path $tmpRoot 'push-build-core.sh' $localScript = @" #!/usr/bin/env bash set -euo pipefail root='$wslRepoRoot' host="`$root/.externals/CLIProxyAPI" binary='$wslBinaryPath' archive='$wslArchivePath' version='$version' commit='$commit' build_date='$buildDate' build_mode='$buildMode' run_tests='$runTests' export GOPROXY=https://goproxy.cn,direct export PATH="`$(go env GOROOT)/bin:`$PATH" if [[ "`$build_mode" == 1 ]]; then cd "`$host" git diff --check if [[ "`$run_tests" == 1 ]]; then go test -race ./internal/pluginhost ./sdk/api/handlers ./sdk/cliproxy/usage -count=1 fi CGO_ENABLED=1 go build -buildvcs=false \ -ldflags="-s -w -X main.Version=`$version -X main.Commit=`$commit -X main.BuildDate=`$build_date" \ -o "`$binary" ./cmd/server/ fi test -f "`$binary" chmod 0755 "`$binary" help_output="`$("`$binary" -h 2>&1)" first_line="`${help_output%%`$'\n'*}" expected_prefix="CLIProxyAPI Version: `$version, Commit: `$commit, BuiltAt: " [[ "`$first_line" == "`$expected_prefix"* ]] stage="`$(mktemp -d)" trap 'rm -rf "`$stage"' EXIT install -m 0755 "`$binary" "`$stage/cli-proxy-api" tar -C "`$stage" -cf - cli-proxy-api | gzip -9 >"`$archive" ( cd "`$(dirname "`$archive")" sha256sum "`$(basename "`$archive")" >"`$(basename "`$archive").sha256" ) printf 'version_line=%s\n' "`$first_line" sha256sum "`$binary" "`$archive" stat -c 'archive_size=%s' "`$archive" "@ Write-Utf8NoBom -Path $localScriptPath -Content $localScript Write-Host "Preparing $version / $commit" $wslLocalScriptPath = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($localScriptPath -replace '\\', '/')) Invoke-NativeCommand $wsl @('-d', $WslDistribution, '--', 'bash', $wslLocalScriptPath) if (-not (Test-Path -LiteralPath $archivePath -PathType Leaf)) { throw "Archive was not created: $archivePath" } if (-not (Test-Path -LiteralPath $checksumPath -PathType Leaf)) { throw "Checksum file was not created: $checksumPath" } $binaryHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $resolvedBinary).Hash.ToLowerInvariant() $archiveHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $archivePath).Hash.ToLowerInvariant() $remoteScriptPath = Join-Path $tmpRoot 'push-remote-core.sh' $remoteScript = @' #!/usr/bin/env bash set -euo pipefail archive_name="$1" expected_archive_hash="$2" expected_binary_hash="$3" expected_version="$4" expected_commit="$5" drain_timeout="$6" drain_poll="$7" root="$8" archive="$root/upload/$archive_name" checksum="$archive.sha256" cpa_script="$root/cpa.sh" tmp='' backup='' rollback_needed=0 manager_was_running=0 keeper_pid='' start_services() { if [[ "$manager_was_running" -eq 1 ]]; then "$cpa_script" start --plus else "$cpa_script" start fi } finish() { rc=$? trap - EXIT if [[ "$rc" -ne 0 && "$rollback_needed" -eq 1 ]]; then printf 'DEPLOYMENT_FAILED_ROLLING_BACK\n' "$cpa_script" stop || true if [[ -n "$backup" && -f "$backup/bin/cli-proxy-api" ]]; then install -o root -g root -m 0755 "$backup/bin/cli-proxy-api" "$root/bin/cli-proxy-api" fi start_services || true fi if [[ -n "$tmp" && -d "$tmp" ]]; then rm -rf "$tmp" fi exit "$rc" } trap finish EXIT printf 'VERIFY_UPLOAD\n' test -f "$archive" test -f "$checksum" actual_archive_hash="$(sha256sum "$archive" | awk '{print $1}')" test "$actual_archive_hash" = "$expected_archive_hash" ( cd "$root/upload" sha256sum -c "$(basename "$checksum")" ) test "$(tar -tzf "$archive")" = 'cli-proxy-api' available_kb="$(df -Pk "$root" | awk 'NR==2 {print $4}')" test "$available_kb" -ge 262144 tmp="$(mktemp -d "$root/.deploy-core.XXXXXX")" tar -xzf "$archive" -C "$tmp" test -f "$tmp/cli-proxy-api" test ! -L "$tmp/cli-proxy-api" chmod 0755 "$tmp/cli-proxy-api" test "$(sha256sum "$tmp/cli-proxy-api" | awk '{print $1}')" = "$expected_binary_hash" help_output="$("$tmp/cli-proxy-api" -h 2>&1)" first_line="${help_output%%$'\n'*}" expected_prefix="CLIProxyAPI Version: $expected_version, Commit: $expected_commit, BuiltAt: " [[ "$first_line" == "$expected_prefix"* ]] printf '%s\n' "$first_line" printf 'PREFLIGHT\n' curl -fsS http://127.0.0.1:8317/healthz >/dev/null admin="$(cat "$root/secrets/cpa-management-key")" curl -fsS -H "Authorization: Bearer $admin" \ http://127.0.0.1:8317/v0/management/plugins >"$tmp/plugins-before.json" python3 - "$tmp/plugins-before.json" "$tmp/plugins-before.normalized.json" <<'PY' import json import sys source, target = sys.argv[1:] payload = json.load(open(source, encoding="utf-8")) items = [] for plugin in payload.get("plugins", []): item = { "id": plugin.get("id"), "path": plugin.get("path"), "version": (plugin.get("metadata") or {}).get("version"), "registered": plugin.get("registered"), "enabled": plugin.get("enabled"), "effective_enabled": plugin.get("effective_enabled"), } if not all((item["registered"], item["enabled"], item["effective_enabled"])): raise SystemExit(f"plugin preflight failed: {item}") items.append(item) items.sort(key=lambda value: value["id"] or "") with open(target, "w", encoding="utf-8") as handle: json.dump(items, handle, sort_keys=True, separators=(",", ":")) print(f"plugins={len(items)}") PY if [[ -s "$root/run/manager.pid" ]] && kill -0 "$(cat "$root/run/manager.pid")" 2>/dev/null; then manager_was_running=1 fi if [[ -s "$root/keeper/run/keeper.pid" ]] && kill -0 "$(cat "$root/keeper/run/keeper.pid")" 2>/dev/null; then keeper_pid="$(cat "$root/keeper/run/keeper.pid")" fi attempts=$((drain_timeout / drain_poll + 1)) active=-1 for attempt in $(seq 1 "$attempts"); do curl -fsS -H "Authorization: Bearer $admin" \ http://127.0.0.1:8317/v0/management/plugins/billing/keys >"$tmp/keys-before.json" active="$(python3 - "$tmp/keys-before.json" <<'PY' import json import sys keys = json.load(open(sys.argv[1], encoding="utf-8")).get("keys", []) print(sum(int((item.get("billing") or {}).get("active_requests") or 0) for item in keys)) PY )" printf 'drain_attempt=%s active_requests=%s\n' "$attempt" "$active" if [[ "$active" -eq 0 ]]; then break fi if [[ "$attempt" -lt "$attempts" ]]; then sleep "$drain_poll" fi done test "$active" -eq 0 backup="$root/backups/$(date -u +%Y%m%dT%H%M%SZ)-core-$expected_version" mkdir "$backup" mkdir -p "$backup/bin" "$backup/data" printf 'STOP_AND_BACKUP\n' rollback_needed=1 "$cpa_script" stop cp -a "$root/bin/cli-proxy-api" "$backup/bin/cli-proxy-api" cp -a "$root/plugins" "$backup/plugins" cp -a "$root/config.yaml" "$backup/config.yaml" find "$root/data" -maxdepth 1 -type f \ \( -name 'billing.db*' -o -name 'cpa-ext.db*' \) \ -exec cp -a -t "$backup/data" {} + ( cd "$root" find plugins -type f -name '*.so' -print0 | sort -z | xargs -0 sha256sum ) >"$backup/plugins.sha256" sed -E 's/^([[:space:]]+secret-key:).*/\1 /' \ "$root/config.yaml" >"$backup/config.normalized.yaml" cp "$tmp/plugins-before.normalized.json" "$backup/plugins.normalized.json" sha256sum "$backup/bin/cli-proxy-api" >"$backup/old-core.sha256" sha256sum "$archive" >"$backup/new-archive.sha256" printf 'INSTALL_CORE\n' install -o root -g root -m 0755 "$tmp/cli-proxy-api" "$root/bin/cli-proxy-api.new" test "$(sha256sum "$root/bin/cli-proxy-api.new" | awk '{print $1}')" = "$expected_binary_hash" mv -f "$root/bin/cli-proxy-api.new" "$root/bin/cli-proxy-api" printf 'START_AND_VERIFY\n' start_services curl -fsS http://127.0.0.1:8317/healthz >/dev/null help_output="$("$root/bin/cli-proxy-api" -h 2>&1)" first_line="${help_output%%$'\n'*}" [[ "$first_line" == "$expected_prefix"* ]] test "$(sha256sum "$root/bin/cli-proxy-api" | awk '{print $1}')" = "$expected_binary_hash" (cd "$root" && sha256sum -c "$backup/plugins.sha256") sed -E 's/^([[:space:]]+secret-key:).*/\1 /' \ "$root/config.yaml" >"$tmp/config-after.normalized.yaml" cmp "$backup/config.normalized.yaml" "$tmp/config-after.normalized.yaml" if [[ -n "$keeper_pid" ]]; then test "$(cat "$root/keeper/run/keeper.pid")" = "$keeper_pid" kill -0 "$keeper_pid" fi if [[ "$manager_was_running" -eq 1 ]]; then test -s "$root/run/manager.pid" kill -0 "$(cat "$root/run/manager.pid")" fi admin="$(cat "$root/secrets/cpa-management-key")" curl -fsS -H "Authorization: Bearer $admin" \ http://127.0.0.1:8317/v0/management/plugins >"$tmp/plugins-after.json" python3 - "$tmp/plugins-after.json" "$tmp/plugins-after.normalized.json" <<'PY' import json import sys source, target = sys.argv[1:] payload = json.load(open(source, encoding="utf-8")) items = [] for plugin in payload.get("plugins", []): item = { "id": plugin.get("id"), "path": plugin.get("path"), "version": (plugin.get("metadata") or {}).get("version"), "registered": plugin.get("registered"), "enabled": plugin.get("enabled"), "effective_enabled": plugin.get("effective_enabled"), } items.append(item) items.sort(key=lambda value: value["id"] or "") with open(target, "w", encoding="utf-8") as handle: json.dump(items, handle, sort_keys=True, separators=(",", ":")) PY cmp "$backup/plugins.normalized.json" "$tmp/plugins-after.normalized.json" grep -q "CLIProxyAPI Version: $expected_version, Commit: $expected_commit" "$root/logs/cpa-console.log" if grep -Eqi 'panic|pluginhost:.*(error|failed)' "$root/logs/cpa-console.log"; then tail -n 100 "$root/logs/cpa-console.log" exit 1 fi rollback_needed=0 printf 'DEPLOYMENT_OK\n' printf 'version=%s\n' "$first_line" printf 'backup=%s\n' "$backup" printf 'core_sha256=%s\n' "$expected_binary_hash" printf 'cpa_pid=%s\n' "$(cat "$root/run/cpa.pid")" if [[ -n "$keeper_pid" ]]; then printf 'keeper_pid=%s\n' "$keeper_pid" fi '@ Write-Utf8NoBom -Path $remoteScriptPath -Content $remoteScript Write-Host "Archive: $archivePath" Write-Host "Archive SHA-256: $archiveHash" Write-Host "Binary SHA-256: $binaryHash" if (-not $PSCmdlet.ShouldProcess("$SshTarget`:$RemoteRoot", "Deploy CPA core $version / $commit")) { return } Invoke-NativeCommand $ssh @('-o', 'BatchMode=yes', $SshTarget, "test -d '$RemoteRoot/upload' -a -x '$RemoteRoot/cpa.sh'") Invoke-NativeCommand $scp @('-C', $archivePath, $checksumPath, "$SshTarget`:$RemoteRoot/upload/") $remoteCommand = "tr -d '\r' | bash -s -- '$archiveName' '$archiveHash' '$binaryHash' '$version' '$commit' '$DrainTimeoutSeconds' '$DrainPollSeconds' '$RemoteRoot'" Get-Content -Raw -LiteralPath $remoteScriptPath | & $ssh '-o' 'BatchMode=yes' $SshTarget $remoteCommand if ($LASTEXITCODE -ne 0) { throw "Remote deployment failed with exit code $LASTEXITCODE." } } finally { Pop-Location }