Files
cpa-plugin/scripts/push.ps1
T

479 lines
16 KiB
PowerShell

[CmdletBinding(SupportsShouldProcess)]
param(
[string]$SshTarget = 'root@akko.pchuan.top',
[string]$RemoteRoot = '/root/cpa',
[string]$WslDistribution = 'Debian',
[int]$DrainTimeoutSeconds = 120,
[int]$DrainPollSeconds = 5,
[string]$BinaryPath = '',
[switch]$SkipBuild,
[switch]$SkipTests,
[switch]$SkipUpstreamCheck
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version Latest
function Resolve-RequiredTool {
param([Parameter(Mandatory)][string]$Name)
$command = Get-Command $Name -ErrorAction SilentlyContinue
if ($null -eq $command) {
throw "Required tool is missing from PATH: $Name"
}
return $command.Source
}
function Invoke-NativeCommand {
param(
[Parameter(Mandatory)][string]$FilePath,
[Parameter(Mandatory)][string[]]$ArgumentList
)
& $FilePath @ArgumentList
if ($LASTEXITCODE -ne 0) {
throw "Command failed with exit code $LASTEXITCODE`: $FilePath $($ArgumentList -join ' ')"
}
}
function Get-NativeOutput {
param(
[Parameter(Mandatory)][string]$FilePath,
[Parameter(Mandatory)][string[]]$ArgumentList
)
$output = & $FilePath @ArgumentList 2>&1
if ($LASTEXITCODE -ne 0) {
throw "Command failed with exit code $LASTEXITCODE`: $FilePath $($ArgumentList -join ' ')`n$($output -join "`n")"
}
return (($output -join "`n").Trim())
}
function Write-Utf8NoBom {
param(
[Parameter(Mandatory)][string]$Path,
[Parameter(Mandatory)][string]$Content
)
$encoding = [System.Text.UTF8Encoding]::new($false)
[System.IO.File]::WriteAllText($Path, ($Content -replace "`r`n", "`n"), $encoding)
}
if ($DrainTimeoutSeconds -lt 0) {
throw 'DrainTimeoutSeconds must be zero or greater.'
}
if ($DrainPollSeconds -lt 1) {
throw 'DrainPollSeconds must be at least one second.'
}
if ($SshTarget -notmatch '^[A-Za-z0-9_.@-]+$') {
throw "Unsupported SSH target format: $SshTarget"
}
if ($RemoteRoot -notmatch '^/[A-Za-z0-9._/-]+$') {
throw "Unsupported remote root format: $RemoteRoot"
}
if ($WslDistribution -notmatch '^[A-Za-z0-9._-]+$') {
throw "Unsupported WSL distribution name: $WslDistribution"
}
$git = Resolve-RequiredTool 'git'
$ssh = Resolve-RequiredTool 'ssh'
$scp = Resolve-RequiredTool 'scp'
$wsl = Resolve-RequiredTool 'wsl.exe'
$repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path
$hostRoot = Join-Path $repoRoot '.externals\CLIProxyAPI'
$runtimeRoot = Join-Path $repoRoot '.runtime'
$tmpRoot = Join-Path $runtimeRoot 'tmp'
$deployRoot = Join-Path $runtimeRoot 'deploy'
if (-not (Test-Path -LiteralPath $hostRoot -PathType Container)) {
throw "CLIProxyAPI submodule is missing: $hostRoot"
}
[System.IO.Directory]::CreateDirectory($tmpRoot) | Out-Null
[System.IO.Directory]::CreateDirectory($deployRoot) | Out-Null
Push-Location $repoRoot
try {
if (-not $SkipUpstreamCheck) {
Invoke-NativeCommand $git @('-C', $hostRoot, 'fetch', '--tags', '--prune', 'origin')
}
$head = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', 'HEAD')
if (-not $SkipUpstreamCheck) {
$originMain = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', 'origin/main')
if ($head -ne $originMain) {
throw "Submodule HEAD is not origin/main. HEAD=$head origin/main=$originMain"
}
}
$tag = Get-NativeOutput $git @('-C', $hostRoot, 'describe', '--tags', '--abbrev=0', $head)
$commit = Get-NativeOutput $git @('-C', $hostRoot, 'rev-parse', '--short=8', $head)
$version = "$tag-dev"
$buildDate = [DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ')
foreach ($value in @($tag, $version, $commit)) {
if ($value -notmatch '^[A-Za-z0-9._+-]+$') {
throw "Unsupported version value: $value"
}
}
Invoke-NativeCommand $git @('-C', $hostRoot, 'diff', '--check')
$expectedModifiedFiles = Get-ChildItem -LiteralPath (Join-Path $repoRoot 'patch') -Filter '*.patch' |
Select-String -Pattern '^diff --git a/([^ ]+) b/' |
ForEach-Object { $_.Matches[0].Groups[1].Value } |
Sort-Object -Unique
$actualModifiedFiles = Get-NativeOutput $git @('-C', $hostRoot, 'diff', '--name-only') |
ForEach-Object { $_ -split "`n" } |
Where-Object { -not [string]::IsNullOrWhiteSpace($_) } |
ForEach-Object { $_.Trim() } |
Sort-Object -Unique
$untrackedHostFiles = Get-NativeOutput $git @('-C', $hostRoot, 'ls-files', '--others', '--exclude-standard')
if (-not [string]::IsNullOrWhiteSpace($untrackedHostFiles)) {
throw "CLIProxyAPI worktree contains untracked files:`n$untrackedHostFiles"
}
$unexpectedChanges = Compare-Object -ReferenceObject $expectedModifiedFiles -DifferenceObject $actualModifiedFiles
if ($unexpectedChanges) {
$details = $unexpectedChanges | ForEach-Object { "$($_.SideIndicator) $($_.InputObject)" }
throw "CLIProxyAPI worktree does not match the files declared by patch/*.patch:`n$($details -join "`n")"
}
$wslRepoRoot = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($repoRoot -replace '\\', '/'))
$archiveName = "cli-proxy-api-$version-$commit-linux-amd64.tar.gz"
$archivePath = Join-Path $deployRoot $archiveName
$checksumPath = "$archivePath.sha256"
$wslArchivePath = "$wslRepoRoot/.runtime/deploy/$archiveName"
if ($SkipBuild) {
if ([string]::IsNullOrWhiteSpace($BinaryPath)) {
$BinaryPath = Join-Path $runtimeRoot 'bin\cli-proxy-api'
}
$resolvedBinary = (Resolve-Path -LiteralPath $BinaryPath).Path
$wslBinaryPath = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($resolvedBinary -replace '\\', '/'))
$buildMode = '0'
} else {
$resolvedBinary = Join-Path $deployRoot "cli-proxy-api-$version-$commit"
$wslBinaryPath = "$wslRepoRoot/.runtime/deploy/cli-proxy-api-$version-$commit"
$buildMode = '1'
}
$runTests = if ($SkipTests) { '0' } else { '1' }
$localScriptPath = Join-Path $tmpRoot 'push-build-core.sh'
$localScript = @"
#!/usr/bin/env bash
set -euo pipefail
root='$wslRepoRoot'
host="`$root/.externals/CLIProxyAPI"
binary='$wslBinaryPath'
archive='$wslArchivePath'
version='$version'
commit='$commit'
build_date='$buildDate'
build_mode='$buildMode'
run_tests='$runTests'
export GOPROXY=https://goproxy.cn,direct
export PATH="`$(go env GOROOT)/bin:`$PATH"
if [[ "`$build_mode" == 1 ]]; then
cd "`$host"
git diff --check
if [[ "`$run_tests" == 1 ]]; then
go test -race ./internal/pluginhost ./sdk/api/handlers ./sdk/cliproxy/usage -count=1
fi
CGO_ENABLED=1 go build -buildvcs=false \
-ldflags="-s -w -X main.Version=`$version -X main.Commit=`$commit -X main.BuildDate=`$build_date" \
-o "`$binary" ./cmd/server/
fi
test -f "`$binary"
chmod 0755 "`$binary"
help_output="`$("`$binary" -h 2>&1)"
first_line="`${help_output%%`$'\n'*}"
expected_prefix="CLIProxyAPI Version: `$version, Commit: `$commit, BuiltAt: "
[[ "`$first_line" == "`$expected_prefix"* ]]
stage="`$(mktemp -d)"
trap 'rm -rf "`$stage"' EXIT
install -m 0755 "`$binary" "`$stage/cli-proxy-api"
tar -C "`$stage" -cf - cli-proxy-api | gzip -9 >"`$archive"
(
cd "`$(dirname "`$archive")"
sha256sum "`$(basename "`$archive")" >"`$(basename "`$archive").sha256"
)
printf 'version_line=%s\n' "`$first_line"
sha256sum "`$binary" "`$archive"
stat -c 'archive_size=%s' "`$archive"
"@
Write-Utf8NoBom -Path $localScriptPath -Content $localScript
Write-Host "Preparing $version / $commit"
$wslLocalScriptPath = Get-NativeOutput $wsl @('-d', $WslDistribution, '--', 'wslpath', '-a', ($localScriptPath -replace '\\', '/'))
Invoke-NativeCommand $wsl @('-d', $WslDistribution, '--', 'bash', $wslLocalScriptPath)
if (-not (Test-Path -LiteralPath $archivePath -PathType Leaf)) {
throw "Archive was not created: $archivePath"
}
if (-not (Test-Path -LiteralPath $checksumPath -PathType Leaf)) {
throw "Checksum file was not created: $checksumPath"
}
$binaryHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $resolvedBinary).Hash.ToLowerInvariant()
$archiveHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $archivePath).Hash.ToLowerInvariant()
$remoteScriptPath = Join-Path $tmpRoot 'push-remote-core.sh'
$remoteScript = @'
#!/usr/bin/env bash
set -euo pipefail
archive_name="$1"
expected_archive_hash="$2"
expected_binary_hash="$3"
expected_version="$4"
expected_commit="$5"
drain_timeout="$6"
drain_poll="$7"
root="$8"
archive="$root/upload/$archive_name"
checksum="$archive.sha256"
cpa_script="$root/cpa.sh"
tmp=''
backup=''
rollback_needed=0
manager_was_running=0
keeper_pid=''
start_services() {
if [[ "$manager_was_running" -eq 1 ]]; then
"$cpa_script" start --plus
else
"$cpa_script" start
fi
}
finish() {
rc=$?
trap - EXIT
if [[ "$rc" -ne 0 && "$rollback_needed" -eq 1 ]]; then
printf 'DEPLOYMENT_FAILED_ROLLING_BACK\n'
"$cpa_script" stop || true
if [[ -n "$backup" && -f "$backup/bin/cli-proxy-api" ]]; then
install -o root -g root -m 0755 "$backup/bin/cli-proxy-api" "$root/bin/cli-proxy-api"
fi
start_services || true
fi
if [[ -n "$tmp" && -d "$tmp" ]]; then
rm -rf "$tmp"
fi
exit "$rc"
}
trap finish EXIT
printf 'VERIFY_UPLOAD\n'
test -f "$archive"
test -f "$checksum"
actual_archive_hash="$(sha256sum "$archive" | awk '{print $1}')"
test "$actual_archive_hash" = "$expected_archive_hash"
(
cd "$root/upload"
sha256sum -c "$(basename "$checksum")"
)
test "$(tar -tzf "$archive")" = 'cli-proxy-api'
available_kb="$(df -Pk "$root" | awk 'NR==2 {print $4}')"
test "$available_kb" -ge 262144
tmp="$(mktemp -d "$root/.deploy-core.XXXXXX")"
tar -xzf "$archive" -C "$tmp"
test -f "$tmp/cli-proxy-api"
test ! -L "$tmp/cli-proxy-api"
chmod 0755 "$tmp/cli-proxy-api"
test "$(sha256sum "$tmp/cli-proxy-api" | awk '{print $1}')" = "$expected_binary_hash"
help_output="$("$tmp/cli-proxy-api" -h 2>&1)"
first_line="${help_output%%$'\n'*}"
expected_prefix="CLIProxyAPI Version: $expected_version, Commit: $expected_commit, BuiltAt: "
[[ "$first_line" == "$expected_prefix"* ]]
printf '%s\n' "$first_line"
printf 'PREFLIGHT\n'
curl -fsS http://127.0.0.1:8317/healthz >/dev/null
admin="$(cat "$root/secrets/cpa-management-key")"
curl -fsS -H "Authorization: Bearer $admin" \
http://127.0.0.1:8317/v0/management/plugins >"$tmp/plugins-before.json"
python3 - "$tmp/plugins-before.json" "$tmp/plugins-before.normalized.json" <<'PY'
import json
import sys
source, target = sys.argv[1:]
payload = json.load(open(source, encoding="utf-8"))
items = []
for plugin in payload.get("plugins", []):
item = {
"id": plugin.get("id"),
"path": plugin.get("path"),
"version": (plugin.get("metadata") or {}).get("version"),
"registered": plugin.get("registered"),
"enabled": plugin.get("enabled"),
"effective_enabled": plugin.get("effective_enabled"),
}
if not all((item["registered"], item["enabled"], item["effective_enabled"])):
raise SystemExit(f"plugin preflight failed: {item}")
items.append(item)
items.sort(key=lambda value: value["id"] or "")
with open(target, "w", encoding="utf-8") as handle:
json.dump(items, handle, sort_keys=True, separators=(",", ":"))
print(f"plugins={len(items)}")
PY
if [[ -s "$root/run/manager.pid" ]] && kill -0 "$(cat "$root/run/manager.pid")" 2>/dev/null; then
manager_was_running=1
fi
if [[ -s "$root/keeper/run/keeper.pid" ]] && kill -0 "$(cat "$root/keeper/run/keeper.pid")" 2>/dev/null; then
keeper_pid="$(cat "$root/keeper/run/keeper.pid")"
fi
attempts=$((drain_timeout / drain_poll + 1))
active=-1
for attempt in $(seq 1 "$attempts"); do
curl -fsS -H "Authorization: Bearer $admin" \
http://127.0.0.1:8317/v0/management/plugins/billing/keys >"$tmp/keys-before.json"
active="$(python3 - "$tmp/keys-before.json" <<'PY'
import json
import sys
keys = json.load(open(sys.argv[1], encoding="utf-8")).get("keys", [])
print(sum(int((item.get("billing") or {}).get("active_requests") or 0) for item in keys))
PY
)"
printf 'drain_attempt=%s active_requests=%s\n' "$attempt" "$active"
if [[ "$active" -eq 0 ]]; then
break
fi
if [[ "$attempt" -lt "$attempts" ]]; then
sleep "$drain_poll"
fi
done
test "$active" -eq 0
backup="$root/backups/$(date -u +%Y%m%dT%H%M%SZ)-core-$expected_version"
mkdir "$backup"
mkdir -p "$backup/bin" "$backup/data"
printf 'STOP_AND_BACKUP\n'
rollback_needed=1
"$cpa_script" stop
cp -a "$root/bin/cli-proxy-api" "$backup/bin/cli-proxy-api"
cp -a "$root/plugins" "$backup/plugins"
cp -a "$root/config.yaml" "$backup/config.yaml"
find "$root/data" -maxdepth 1 -type f \
\( -name 'billing.db*' -o -name 'cpa-ext.db*' \) \
-exec cp -a -t "$backup/data" {} +
(
cd "$root"
find plugins -type f -name '*.so' -print0 | sort -z | xargs -0 sha256sum
) >"$backup/plugins.sha256"
sed -E 's/^([[:space:]]+secret-key:).*/\1 <normalized>/' \
"$root/config.yaml" >"$backup/config.normalized.yaml"
cp "$tmp/plugins-before.normalized.json" "$backup/plugins.normalized.json"
sha256sum "$backup/bin/cli-proxy-api" >"$backup/old-core.sha256"
sha256sum "$archive" >"$backup/new-archive.sha256"
printf 'INSTALL_CORE\n'
install -o root -g root -m 0755 "$tmp/cli-proxy-api" "$root/bin/cli-proxy-api.new"
test "$(sha256sum "$root/bin/cli-proxy-api.new" | awk '{print $1}')" = "$expected_binary_hash"
mv -f "$root/bin/cli-proxy-api.new" "$root/bin/cli-proxy-api"
printf 'START_AND_VERIFY\n'
start_services
curl -fsS http://127.0.0.1:8317/healthz >/dev/null
help_output="$("$root/bin/cli-proxy-api" -h 2>&1)"
first_line="${help_output%%$'\n'*}"
[[ "$first_line" == "$expected_prefix"* ]]
test "$(sha256sum "$root/bin/cli-proxy-api" | awk '{print $1}')" = "$expected_binary_hash"
(cd "$root" && sha256sum -c "$backup/plugins.sha256")
sed -E 's/^([[:space:]]+secret-key:).*/\1 <normalized>/' \
"$root/config.yaml" >"$tmp/config-after.normalized.yaml"
cmp "$backup/config.normalized.yaml" "$tmp/config-after.normalized.yaml"
if [[ -n "$keeper_pid" ]]; then
test "$(cat "$root/keeper/run/keeper.pid")" = "$keeper_pid"
kill -0 "$keeper_pid"
fi
if [[ "$manager_was_running" -eq 1 ]]; then
test -s "$root/run/manager.pid"
kill -0 "$(cat "$root/run/manager.pid")"
fi
admin="$(cat "$root/secrets/cpa-management-key")"
curl -fsS -H "Authorization: Bearer $admin" \
http://127.0.0.1:8317/v0/management/plugins >"$tmp/plugins-after.json"
python3 - "$tmp/plugins-after.json" "$tmp/plugins-after.normalized.json" <<'PY'
import json
import sys
source, target = sys.argv[1:]
payload = json.load(open(source, encoding="utf-8"))
items = []
for plugin in payload.get("plugins", []):
item = {
"id": plugin.get("id"),
"path": plugin.get("path"),
"version": (plugin.get("metadata") or {}).get("version"),
"registered": plugin.get("registered"),
"enabled": plugin.get("enabled"),
"effective_enabled": plugin.get("effective_enabled"),
}
items.append(item)
items.sort(key=lambda value: value["id"] or "")
with open(target, "w", encoding="utf-8") as handle:
json.dump(items, handle, sort_keys=True, separators=(",", ":"))
PY
cmp "$backup/plugins.normalized.json" "$tmp/plugins-after.normalized.json"
grep -q "CLIProxyAPI Version: $expected_version, Commit: $expected_commit" "$root/logs/cpa-console.log"
if grep -Eqi 'panic|pluginhost:.*(error|failed)' "$root/logs/cpa-console.log"; then
tail -n 100 "$root/logs/cpa-console.log"
exit 1
fi
rollback_needed=0
printf 'DEPLOYMENT_OK\n'
printf 'version=%s\n' "$first_line"
printf 'backup=%s\n' "$backup"
printf 'core_sha256=%s\n' "$expected_binary_hash"
printf 'cpa_pid=%s\n' "$(cat "$root/run/cpa.pid")"
if [[ -n "$keeper_pid" ]]; then
printf 'keeper_pid=%s\n' "$keeper_pid"
fi
'@
Write-Utf8NoBom -Path $remoteScriptPath -Content $remoteScript
Write-Host "Archive: $archivePath"
Write-Host "Archive SHA-256: $archiveHash"
Write-Host "Binary SHA-256: $binaryHash"
if (-not $PSCmdlet.ShouldProcess("$SshTarget`:$RemoteRoot", "Deploy CPA core $version / $commit")) {
return
}
Invoke-NativeCommand $ssh @('-o', 'BatchMode=yes', $SshTarget, "test -d '$RemoteRoot/upload' -a -x '$RemoteRoot/cpa.sh'")
Invoke-NativeCommand $scp @('-C', $archivePath, $checksumPath, "$SshTarget`:$RemoteRoot/upload/")
$remoteCommand = "tr -d '\r' | bash -s -- '$archiveName' '$archiveHash' '$binaryHash' '$version' '$commit' '$DrainTimeoutSeconds' '$DrainPollSeconds' '$RemoteRoot'"
Get-Content -Raw -LiteralPath $remoteScriptPath | & $ssh '-o' 'BatchMode=yes' $SshTarget $remoteCommand
if ($LASTEXITCODE -ne 0) {
throw "Remote deployment failed with exit code $LASTEXITCODE."
}
} finally {
Pop-Location
}