diff --git a/.env.full.example b/.env.full.example index edd28b6..9c6403d 100644 --- a/.env.full.example +++ b/.env.full.example @@ -4,8 +4,20 @@ # 1. 若国内服务器无法直接拉官方镜像,可把下列镜像名改成您已缓存或私有仓库中的地址。 # 2. Compose 默认读取根目录 .env;请按需复制为 .env 后再启动。 -SMARTFLOW_BACKEND_IMAGE=smartflow/backend-suite:latest -SMARTFLOW_FRONTEND_IMAGE=smartflow/frontend:latest +SMARTFLOW_IMAGE_USERAUTH=smartflow/userauth:latest +SMARTFLOW_IMAGE_NOTIFICATION=smartflow/notification:latest +SMARTFLOW_IMAGE_ACTIVE_SCHEDULER=smartflow/active-scheduler:latest +SMARTFLOW_IMAGE_SCHEDULE=smartflow/schedule:latest +SMARTFLOW_IMAGE_TASK=smartflow/task:latest +SMARTFLOW_IMAGE_TASK_CLASS=smartflow/task-class:latest +SMARTFLOW_IMAGE_COURSE=smartflow/course:latest +SMARTFLOW_IMAGE_MEMORY=smartflow/memory:latest +SMARTFLOW_IMAGE_AGENT=smartflow/agent:latest +SMARTFLOW_IMAGE_TASKCLASSFORUM=smartflow/taskclassforum:latest +SMARTFLOW_IMAGE_TOKENSTORE=smartflow/tokenstore:latest +SMARTFLOW_IMAGE_LLM=smartflow/llm:latest +SMARTFLOW_IMAGE_API=smartflow/api:latest +SMARTFLOW_IMAGE_FRONTEND=smartflow/frontend:latest ARK_API_KEY= SMARTFLOW_USERAUTH_ALLOWREGISTER=false SMARTFLOW_NOTIFICATION_FRONTENDBASEURL=https://smartflow.example.com diff --git a/.gitea/workflows/release-offline.yml b/.gitea/workflows/release-offline.yml index 022fe13..f43ff62 100644 --- a/.gitea/workflows/release-offline.yml +++ b/.gitea/workflows/release-offline.yml @@ -12,111 +12,207 @@ on: default: "false" jobs: - package-and-deploy: - runs-on: build-host + build-upload: + runs-on: local-build steps: - name: Prepare local worktree env: - SMARTFLOW_REPO_SLUG: ${{ gitea.repository }} + SMARTFLOW_REPO_URL: https://git.lecspace.com/${{ gitea.repository }}.git + SMARTFLOW_GIT_REPO_URL: ${{ secrets.SMARTFLOW_GIT_REPO_URL }} + SMARTFLOW_REPO_SHA: ${{ gitea.sha }} + SMARTFLOW_GITEA_USER: ${{ secrets.SMARTFLOW_GITEA_USER }} + SMARTFLOW_GITEA_TOKEN: ${{ secrets.SMARTFLOW_GITEA_TOKEN }} + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + function Add-GitHubEnv { + param([string]$Line) + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::AppendAllText($env:GITHUB_ENV, $Line + [Environment]::NewLine, $utf8NoBom) + } + + $worktreeRoot = Join-Path ([System.IO.Path]::GetTempPath()) "smartflow-actions" + $worktree = Join-Path $worktreeRoot $env:SMARTFLOW_REPO_SHA + $repoUrl = if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_GIT_REPO_URL)) { $env:SMARTFLOW_REPO_URL } else { $env:SMARTFLOW_GIT_REPO_URL } + + if (Test-Path $worktree) { + Remove-Item -LiteralPath $worktree -Recurse -Force + } + New-Item -ItemType Directory -Force -Path $worktreeRoot | Out-Null + + $gitArgs = @() + if (-not [string]::IsNullOrWhiteSpace($env:SMARTFLOW_GITEA_TOKEN)) { + $giteaUser = $env:SMARTFLOW_GITEA_USER + if ([string]::IsNullOrWhiteSpace($giteaUser)) { $giteaUser = "Losita" } + $basicToken = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $giteaUser, $env:SMARTFLOW_GITEA_TOKEN))) + $gitArgs += @("-c", ("http.extraHeader=Authorization: Basic {0}" -f $basicToken)) + } + + & git @gitArgs clone --no-checkout $repoUrl $worktree + if ($LASTEXITCODE -ne 0) { throw "source clone failed." } + + & git -C $worktree checkout --force $env:SMARTFLOW_REPO_SHA + if ($LASTEXITCODE -ne 0) { throw "source checkout failed." } + + & git -C $worktree clean -dffx + if ($LASTEXITCODE -ne 0) { throw "source cleanup failed." } + + $appTag = (& git -C $worktree rev-parse --short=12 HEAD).Trim() + Add-GitHubEnv "APP_TAG=$appTag" + Add-GitHubEnv "SMARTFLOW_WORKTREE=$worktree" + + - name: Resolve release base + env: + INPUT_BASE_REF: ${{ inputs.base_ref }} + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + function Add-GitHubEnv { + param([string]$Line) + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::AppendAllText($env:GITHUB_ENV, $Line + [Environment]::NewLine, $utf8NoBom) + } + + Set-Location $env:SMARTFLOW_WORKTREE + $baseRef = $env:INPUT_BASE_REF + if ([string]::IsNullOrWhiteSpace($baseRef)) { + & git rev-parse --verify --quiet "HEAD^" | Out-Null + if ($LASTEXITCODE -eq 0) { + $baseRef = (& git rev-parse "HEAD^").Trim() + } + } + + Add-GitHubEnv "BASE_REF=$baseRef" + + - name: Build release plan + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + Set-Location $env:SMARTFLOW_WORKTREE + .\deploy\impact-rules.ps1 -BaseRef $env:BASE_REF -HeadRef "HEAD" -OutputFile "deploy\release-plan.env" + Get-Content -LiteralPath "deploy\release-plan.env" + + - name: Pack docker images + env: + INPUT_INCLUDE_INFRA: ${{ inputs.include_infra }} + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + Set-Location $env:SMARTFLOW_WORKTREE + $packArgs = @("-AppTag", $env:APP_TAG, "-PlanFile", "deploy\release-plan.env") + if ($env:INPUT_INCLUDE_INFRA -eq "true") { + $packArgs += "-IncludeInfra" + } + .\deploy\docker-pack.ps1 @packArgs + + - name: Stage release directory + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + Set-Location $env:SMARTFLOW_WORKTREE + .\deploy\stage-release.ps1 ` + -ReleaseDir ".release\$env:APP_TAG" ` + -PlanFile "deploy\release-plan.env" ` + -BundleDir ".docker-bundles" + + - name: Upload release to server + env: + SMARTFLOW_RELEASE_HOST: ${{ secrets.SMARTFLOW_RELEASE_HOST }} + SMARTFLOW_RELEASE_USER: ${{ secrets.SMARTFLOW_RELEASE_USER }} + SMARTFLOW_RELEASE_PORT: ${{ secrets.SMARTFLOW_RELEASE_PORT }} + SMARTFLOW_RELEASE_ROOT: ${{ secrets.SMARTFLOW_RELEASE_ROOT }} + SMARTFLOW_SSH_KEY: ${{ secrets.SMARTFLOW_SSH_KEY }} + shell: powershell + run: | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version Latest + + Set-Location $env:SMARTFLOW_WORKTREE + $hostName = $env:SMARTFLOW_RELEASE_HOST + if ([string]::IsNullOrWhiteSpace($hostName)) { $hostName = "192.140.166.210" } + $userName = $env:SMARTFLOW_RELEASE_USER + if ([string]::IsNullOrWhiteSpace($userName)) { $userName = "root" } + $port = $env:SMARTFLOW_RELEASE_PORT + if ([string]::IsNullOrWhiteSpace($port)) { $port = "22" } + $releaseRoot = $env:SMARTFLOW_RELEASE_ROOT + if ([string]::IsNullOrWhiteSpace($releaseRoot)) { $releaseRoot = "/srv/smartflow/releases" } + if ($releaseRoot -notmatch '^/srv/smartflow/releases(/.*)?$') { throw "release root must stay under /srv/smartflow/releases." } + $remote = "{0}@{1}" -f $userName, $hostName + $archivePath = Join-Path ([System.IO.Path]::GetTempPath()) ("smartflow-release-{0}.tgz" -f $env:APP_TAG) + $remoteArchive = ("{0}/{1}.tgz" -f $releaseRoot.TrimEnd('/'), $env:APP_TAG) + + if (Test-Path $archivePath) { + Remove-Item -LiteralPath $archivePath -Force + } + & tar -C ".release\$env:APP_TAG" -czf $archivePath . + if ($LASTEXITCODE -ne 0) { throw "release archive failed." } + + $sshArgs = @("-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=30", "-p", $port) + $scpArgs = @("-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=30", "-P", $port) + if (-not [string]::IsNullOrWhiteSpace($env:SMARTFLOW_SSH_KEY)) { + $keyPath = Join-Path ([System.IO.Path]::GetTempPath()) ("smartflow-release-{0}.key" -f $env:APP_TAG) + $env:SMARTFLOW_SSH_KEY.Replace("`r`n", "`n") | Out-File -FilePath $keyPath -Encoding ascii -NoNewline + if ([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform([System.Runtime.InteropServices.OSPlatform]::Windows)) { + & icacls $keyPath /inheritance:r /grant:r "$($env:USERNAME):(R)" | Out-Null + } else { + & chmod 600 $keyPath + } + $sshArgs += @("-i", $keyPath) + $scpArgs += @("-i", $keyPath) + } + + & ssh @sshArgs $remote "mkdir -p '$releaseRoot'" + if ($LASTEXITCODE -ne 0) { throw "remote release root prepare failed." } + + & scp @scpArgs $archivePath ("{0}:{1}" -f $remote, $remoteArchive) + if ($LASTEXITCODE -ne 0) { throw "release upload failed." } + + $remoteScript = @( + "set -euo pipefail", + "release_root='$releaseRoot'", + "app_tag='$env:APP_TAG'", + "archive='$remoteArchive'", + "[[ -n `"`$release_root`" && `"`$app_tag`" =~ ^[0-9a-f]{12}$ ]]", + "target=`"`$release_root/`$app_tag`"", + "rm -rf `"`$target`"", + "mkdir -p `"`$target`"", + "tar -xzf `"`$archive`" -C `"`$target`"", + "rm -f `"`$archive`"" + ) -join "`n" + + $remoteScript | ssh @sshArgs $remote "bash -s" + if ($LASTEXITCODE -ne 0) { throw "remote release unpack failed." } + + - name: Cleanup worktree + if: ${{ always() }} + shell: powershell + run: | + $ErrorActionPreference = "Stop" + $worktreeRoot = Join-Path ([System.IO.Path]::GetTempPath()) "smartflow-actions" + $expectedPrefix = $worktreeRoot.TrimEnd([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar + if (-not [string]::IsNullOrWhiteSpace($env:SMARTFLOW_WORKTREE) -and $env:SMARTFLOW_WORKTREE.StartsWith($expectedPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + Remove-Item -LiteralPath $env:SMARTFLOW_WORKTREE -Recurse -Force -ErrorAction SilentlyContinue + } + + deploy: + runs-on: build-host + needs: build-upload + steps: + - name: Trigger deploy + env: SMARTFLOW_REPO_SHA: ${{ gitea.sha }} shell: bash run: | set -euo pipefail - repo_slug="${SMARTFLOW_REPO_SLUG,,}" - bare_repo="/srv/gitea/data/gitea/data/gitea-repositories/${repo_slug}.git" - worktree_root="/tmp/smartflow-actions" - worktree="${worktree_root}/${SMARTFLOW_REPO_SHA}" - if [[ ! -d "${bare_repo}" ]]; then - echo "gitea bare repo not found: ${bare_repo}" >&2 - exit 65 - fi - export HOME="${HOME:-${worktree_root}/home}" - mkdir -p "${HOME}" - git config --global --add safe.directory "${bare_repo}" - rm -rf "${worktree}" - mkdir -p "${worktree_root}" - git clone --no-checkout "${bare_repo}" "${worktree}" - git -C "${worktree}" checkout --force "${SMARTFLOW_REPO_SHA}" - git -C "${worktree}" clean -dffx - app_tag="$(git -C "${worktree}" rev-parse --short=12 HEAD)" - { - echo "APP_TAG=${app_tag}" - echo "SMARTFLOW_WORKTREE=${worktree}" - } >> "${GITHUB_ENV}" - - - name: Resolve release base - shell: bash - run: | - set -euo pipefail - cd "${SMARTFLOW_WORKTREE}" - BASE_REF="${{ inputs.base_ref }}" - if [[ -z "${BASE_REF}" ]] && git rev-parse --verify --quiet HEAD^ >/dev/null; then - BASE_REF="$(git rev-parse HEAD^)" - fi - echo "BASE_REF=${BASE_REF}" >> "${GITHUB_ENV}" - - - name: Build release plan - shell: bash - run: | - set -euo pipefail - cd "${SMARTFLOW_WORKTREE}" - ./deploy/impact-rules.sh "${BASE_REF:-}" HEAD deploy/release-plan.env - cat deploy/release-plan.env - - - name: Pack docker images - shell: bash - run: | - set -euo pipefail - cd "${SMARTFLOW_WORKTREE}" - source deploy/release-plan.env - args=(--app-tag "${APP_TAG}") - if [[ "${SMARTFLOW_BUILD_BACKEND}" != "1" ]]; then - args+=(--skip-backend) - fi - if [[ "${SMARTFLOW_BUILD_FRONTEND}" != "1" ]]; then - args+=(--skip-frontend) - fi - if [[ "${{ inputs.include_infra }}" == "true" ]]; then - args+=(--include-infra) - fi - ./deploy/docker-pack.sh "${args[@]}" - - - name: Stage release directory - shell: bash - run: | - set -euo pipefail - cd "${SMARTFLOW_WORKTREE}" - ./deploy/stage-release.sh \ - --release-dir ".release/${APP_TAG}" \ - --plan-file "deploy/release-plan.env" \ - --bundle-dir ".docker-bundles" - - - name: Materialize release directory - shell: bash - run: | - set -euo pipefail - cd "${SMARTFLOW_WORKTREE}" - release_root="/srv/smartflow/releases/${APP_TAG}" - release_archive="/srv/smartflow/releases/${APP_TAG}.tgz" - mkdir -p /srv/smartflow/releases - rm -f "${release_archive}" - tar -C ".release/${APP_TAG}" -czf "${release_archive}" . - rm -rf "${release_root}" - mkdir -p "${release_root}" - tar -xzf "${release_archive}" -C "${release_root}" - rm -f "${release_archive}" - - - name: Trigger deploy - shell: bash - run: | - set -euo pipefail - smartflow-release deploy "${APP_TAG}" - - - name: Cleanup worktree - if: ${{ always() }} - shell: bash - run: | - set -euo pipefail - if [[ -n "${SMARTFLOW_WORKTREE:-}" && "${SMARTFLOW_WORKTREE}" == /tmp/smartflow-actions/* ]]; then - rm -rf "${SMARTFLOW_WORKTREE}" - fi + app_tag="${SMARTFLOW_REPO_SHA:0:12}" + smartflow-release deploy "${app_tag}" diff --git a/backend/Dockerfile b/backend/Dockerfile index 115941e..73ff53c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -15,6 +15,8 @@ COPY . . ARG TARGETOS=linux ARG TARGETARCH=amd64 +FROM builder AS suite-builder + # 1. 统一构建所有需要部署的后端服务二进制,避免每个服务维护一份 Dockerfile。 # 2. 输出目录固定为 /out,便于运行时镜像按命令复用同一套产物。 RUN --mount=type=cache,target=/root/.cache/go-build \ @@ -23,7 +25,17 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o "/out/${service}" "./cmd/${service}"; \ done -FROM ${RUNTIME_IMAGE} AS runtime +FROM builder AS service-builder + +ARG SERVICE=api + +# 1. 服务级镜像只编译一个入口,减少单服务发布时需要上传的二进制体积。 +# 2. SERVICE 必须对应 backend/cmd 下的目录;构建失败会直接暴露错误,避免发布错误镜像。 +RUN --mount=type=cache,target=/root/.cache/go-build \ + mkdir -p /out && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o "/out/${SERVICE}" "./cmd/${SERVICE}" + +FROM ${RUNTIME_IMAGE} AS runtime-base WORKDIR /app/backend @@ -33,10 +45,23 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates tzdata \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /out /app/bin COPY config.docker.yaml /app/backend/config.docker.yaml ENV TZ=Asia/Shanghai ENV SMARTFLOW_CONFIG_FILE=/app/backend/config.docker.yaml +FROM runtime-base AS runtime-suite + +COPY --from=suite-builder /out /app/bin + CMD ["/app/bin/api"] + +FROM runtime-base AS runtime-service + +ARG SERVICE=api + +COPY --from=service-builder /out/${SERVICE} /app/bin/${SERVICE} + +CMD ["/app/bin/api"] + +FROM runtime-suite AS runtime diff --git a/deploy/docker-pack.ps1 b/deploy/docker-pack.ps1 index 1bb26b8..444cee0 100644 --- a/deploy/docker-pack.ps1 +++ b/deploy/docker-pack.ps1 @@ -1,90 +1,177 @@ param( [string]$AppTag = "latest", - [string]$BackendImage = "smartflow/backend-suite", - [string]$FrontendImage = "smartflow/frontend", [string]$OutputDir = ".docker-bundles", - [switch]$IncludeInfra + [string]$PlanFile = "", + [string]$Services = "", + [switch]$IncludeInfra, + [switch]$SkipBackend, + [switch]$SkipFrontend, + [string]$BackendImage = "", + [string]$FrontendImage = "" ) $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest +. (Join-Path $PSScriptRoot "service-catalog.ps1") -function Get-ImageRef { - param( - [string]$EnvName, - [string]$DefaultValue - ) +function Read-ReleasePlan { + param([string]$Path) - $value = [Environment]::GetEnvironmentVariable($EnvName) - if ([string]::IsNullOrWhiteSpace($value)) { - return $DefaultValue + $values = @{} + if ([string]::IsNullOrWhiteSpace($Path)) { + return $values + } + if (-not (Test-Path -LiteralPath $Path)) { + throw ("release plan not found: {0}" -f $Path) } - return $value.Trim() + foreach ($line in Get-Content -LiteralPath $Path -Encoding UTF8) { + if ([string]::IsNullOrWhiteSpace($line) -or $line.TrimStart().StartsWith("#")) { + continue + } + + $parts = $line -split "=", 2 + if ($parts.Count -ne 2) { + throw ("invalid release plan line: {0}" -f $line) + } + $values[$parts[0].Trim()] = $parts[1].Trim() + } + + return $values +} + +function Get-ImageRefForService { + param( + [string]$Service, + [hashtable]$Plan, + [string]$Tag + ) + + $imageEnv = Get-SmartFlowImageEnvForService -Service $Service + if ($Plan.ContainsKey($imageEnv) -and -not [string]::IsNullOrWhiteSpace($Plan[$imageEnv])) { + return $Plan[$imageEnv] + } + + return Get-SmartFlowDefaultImageForService -Service $Service -AppTag $Tag +} + +function Invoke-Docker { + param([string[]]$Arguments) + + & docker @Arguments + if ($LASTEXITCODE -ne 0) { + throw ("docker command failed: docker {0}" -f ($Arguments -join " ")) + } } $repoRoot = Split-Path -Parent $PSScriptRoot $bundleDir = Join-Path $repoRoot $OutputDir -$backendRef = "{0}:{1}" -f $BackendImage, $AppTag -$frontendRef = "{0}:{1}" -f $FrontendImage, $AppTag -$appBundlePath = Join-Path $bundleDir ("smartflow-app-{0}.tar" -f $AppTag) -$infraBundlePath = Join-Path $bundleDir ("smartflow-infra-{0}.tar" -f $AppTag) +$plan = Read-ReleasePlan -Path $PlanFile -New-Item -ItemType Directory -Force -Path $bundleDir | Out-Null - -Write-Host "==> Build backend image $backendRef" -docker build --platform linux/amd64 -f (Join-Path $repoRoot "backend\Dockerfile") -t $backendRef (Join-Path $repoRoot "backend") -if ($LASTEXITCODE -ne 0) { - throw "Backend image build failed." +if ($plan.ContainsKey("SMARTFLOW_APP_TAG") -and -not [string]::IsNullOrWhiteSpace($plan["SMARTFLOW_APP_TAG"])) { + $AppTag = $plan["SMARTFLOW_APP_TAG"] } -Write-Host "==> Build frontend image $frontendRef" -docker build --platform linux/amd64 -f (Join-Path $repoRoot "frontend\Dockerfile") -t $frontendRef (Join-Path $repoRoot "frontend") -if ($LASTEXITCODE -ne 0) { - throw "Frontend image build failed." +if ([string]::IsNullOrWhiteSpace($Services) -and $plan.ContainsKey("SMARTFLOW_RESTART_SERVICES")) { + $Services = $plan["SMARTFLOW_RESTART_SERVICES"] } -if (Test-Path $appBundlePath) { - Remove-Item -LiteralPath $appBundlePath -Force -} - -Write-Host "==> Export app bundle to $appBundlePath" -docker save -o $appBundlePath $backendRef $frontendRef -if ($LASTEXITCODE -ne 0) { - throw "App bundle export failed." -} - -if (-not $IncludeInfra) { - Write-Host "==> Done. App bundle exported." - return -} - -$infraImages = @( - (Get-ImageRef -EnvName "SMARTFLOW_MYSQL_IMAGE" -DefaultValue "mysql:8.0"), - (Get-ImageRef -EnvName "SMARTFLOW_REDIS_IMAGE" -DefaultValue "redis:7"), - (Get-ImageRef -EnvName "SMARTFLOW_KAFKA_IMAGE" -DefaultValue "apache/kafka:3.7.2"), - (Get-ImageRef -EnvName "SMARTFLOW_ETCD_IMAGE" -DefaultValue "quay.io/coreos/etcd:v3.5.5"), - (Get-ImageRef -EnvName "SMARTFLOW_MINIO_IMAGE" -DefaultValue "minio/minio:RELEASE.2023-03-20T20-16-18Z"), - (Get-ImageRef -EnvName "SMARTFLOW_MILVUS_IMAGE" -DefaultValue "milvusdb/milvus:v2.4.4"), - (Get-ImageRef -EnvName "SMARTFLOW_ATTU_IMAGE" -DefaultValue "zilliz/attu:v2.4.3") -) - -foreach ($imageRef in $infraImages) { - Write-Host "==> Pull infra image $imageRef" - docker pull $imageRef - if ($LASTEXITCODE -ne 0) { - throw ("Infra image pull failed: {0}" -f $imageRef) +$selectedServices = @() +if (-not [string]::IsNullOrWhiteSpace($Services)) { + $selectedServices = @($Services.Split(",") | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim() }) +} elseif ([string]::IsNullOrWhiteSpace($PlanFile)) { + if (-not $SkipBackend) { + $selectedServices += @(Get-SmartFlowBackendServices) + } + if (-not $SkipFrontend) { + $selectedServices += "frontend" } } -if (Test-Path $infraBundlePath) { +New-Item -ItemType Directory -Force -Path $bundleDir | Out-Null +$appBundlePath = Join-Path $bundleDir ("smartflow-app-{0}.tar" -f $AppTag) +$infraBundlePath = Join-Path $bundleDir ("smartflow-infra-{0}.tar" -f $AppTag) + +if (Test-Path -LiteralPath $appBundlePath) { + Remove-Item -LiteralPath $appBundlePath -Force +} + +$appImages = @() +foreach ($service in $selectedServices) { + if ($service -eq "frontend") { + if ($SkipFrontend) { + continue + } + + $imageRef = Get-ImageRefForService -Service $service -Plan $plan -Tag $AppTag + Write-Host "==> Build frontend image $imageRef" + Invoke-Docker -Arguments @( + "build", "--platform", "linux/amd64", + "-f", (Join-Path $repoRoot "frontend\Dockerfile"), + "-t", $imageRef, + (Join-Path $repoRoot "frontend") + ) + $appImages += $imageRef + continue + } + + if ($SkipBackend) { + continue + } + if (-not (Test-SmartFlowBackendService -Service $service)) { + throw ("unknown backend release service: {0}" -f $service) + } + + $imageRef = Get-ImageRefForService -Service $service -Plan $plan -Tag $AppTag + Write-Host "==> Build backend service image $imageRef" + Invoke-Docker -Arguments @( + "build", "--platform", "linux/amd64", + "--target", "runtime-service", + "--build-arg", ("SERVICE={0}" -f $service), + "-f", (Join-Path $repoRoot "backend\Dockerfile"), + "-t", $imageRef, + (Join-Path $repoRoot "backend") + ) + $appImages += $imageRef +} + +if ($appImages.Count -gt 0) { + Write-Host "==> Export app bundle to $appBundlePath" + Invoke-Docker -Arguments (@("save", "-o", $appBundlePath) + $appImages) +} else { + Write-Host "==> Skip app bundle export because no application image is selected" +} + +if (-not $IncludeInfra) { + Write-Host "==> Done." + return +} + +$infraImages = @() +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_MYSQL_IMAGE)) { $infraImages += "mysql:8.0" } else { $infraImages += $env:SMARTFLOW_MYSQL_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_REDIS_IMAGE)) { $infraImages += "redis:7" } else { $infraImages += $env:SMARTFLOW_REDIS_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_KAFKA_IMAGE)) { $infraImages += "apache/kafka:3.7.2" } else { $infraImages += $env:SMARTFLOW_KAFKA_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_ETCD_IMAGE)) { $infraImages += "quay.io/coreos/etcd:v3.5.5" } else { $infraImages += $env:SMARTFLOW_ETCD_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_MINIO_IMAGE)) { $infraImages += "minio/minio:RELEASE.2023-03-20T20-16-18Z" } else { $infraImages += $env:SMARTFLOW_MINIO_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_MILVUS_IMAGE)) { $infraImages += "milvusdb/milvus:v2.4.4" } else { $infraImages += $env:SMARTFLOW_MILVUS_IMAGE } +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_ATTU_IMAGE)) { $infraImages += "zilliz/attu:v2.4.3" } else { $infraImages += $env:SMARTFLOW_ATTU_IMAGE } + +foreach ($imageRef in $infraImages) { + & docker image inspect $imageRef | Out-Null + if ($LASTEXITCODE -eq 0) { + Write-Host "==> Reuse local infra image $imageRef" + continue + } + + Write-Host "==> Pull infra image $imageRef" + Invoke-Docker -Arguments @("pull", $imageRef) +} + +if (Test-Path -LiteralPath $infraBundlePath) { Remove-Item -LiteralPath $infraBundlePath -Force } Write-Host "==> Export infra bundle to $infraBundlePath" -docker save -o $infraBundlePath @infraImages -if ($LASTEXITCODE -ne 0) { - throw "Infra bundle export failed." -} +Invoke-Docker -Arguments (@("save", "-o", $infraBundlePath) + $infraImages) -Write-Host "==> Done. App bundle and infra bundle exported." +Write-Host "==> Done." diff --git a/deploy/docker-pack.sh b/deploy/docker-pack.sh index d5e4278..0a54141 100755 --- a/deploy/docker-pack.sh +++ b/deploy/docker-pack.sh @@ -2,10 +2,10 @@ set -euo pipefail APP_TAG="latest" -BACKEND_IMAGE="smartflow/backend-suite" -FRONTEND_IMAGE="smartflow/frontend" OUTPUT_DIR=".docker-bundles" INCLUDE_INFRA=0 +PLAN_FILE="" +SERVICES_CSV="" SKIP_BACKEND=0 SKIP_FRONTEND=0 @@ -15,14 +15,6 @@ while [[ $# -gt 0 ]]; do APP_TAG="$2" shift 2 ;; - --backend-image) - BACKEND_IMAGE="$2" - shift 2 - ;; - --frontend-image) - FRONTEND_IMAGE="$2" - shift 2 - ;; --output-dir) OUTPUT_DIR="$2" shift 2 @@ -31,6 +23,14 @@ while [[ $# -gt 0 ]]; do INCLUDE_INFRA=1 shift ;; + --plan-file) + PLAN_FILE="$2" + shift 2 + ;; + --services) + SERVICES_CSV="$2" + shift 2 + ;; --skip-backend) SKIP_BACKEND=1 shift @@ -39,6 +39,10 @@ while [[ $# -gt 0 ]]; do SKIP_FRONTEND=1 shift ;; + --backend-image|--frontend-image) + # 兼容旧调用参数。服务级发布后镜像引用来自 release-plan.env,不再由统一 backend/frontend 参数决定。 + shift 2 + ;; *) echo "unknown argument: $1" >&2 exit 64 @@ -47,38 +51,100 @@ while [[ $# -gt 0 ]]; do done repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "${repo_root}/deploy/service-catalog.sh" + bundle_dir="${repo_root}/${OUTPUT_DIR}" -backend_ref="${BACKEND_IMAGE}:${APP_TAG}" -frontend_ref="${FRONTEND_IMAGE}:${APP_TAG}" app_bundle_path="${bundle_dir}/smartflow-app-${APP_TAG}.tar" infra_bundle_path="${bundle_dir}/smartflow-infra-${APP_TAG}.tar" mkdir -p "${bundle_dir}" +rm -f "${app_bundle_path}" -if [[ "${SKIP_BACKEND}" -eq 0 ]]; then - echo "==> Build backend image ${backend_ref}" - docker build --platform linux/amd64 -f "${repo_root}/backend/Dockerfile" -t "${backend_ref}" "${repo_root}/backend" +if [[ -n "${PLAN_FILE}" ]]; then + # 1. 构建机只信任影响分析生成的计划文件,避免 workflow 和脚本重复计算影响范围。 + # 2. plan 文件缺失时直接失败,防止误把默认全量构建当成精准发布。 + # 3. SMARTFLOW_APP_TAG 优先级高于命令行参数,保证镜像 tag 与 release id 一致。 + source "${PLAN_FILE}" + APP_TAG="${SMARTFLOW_APP_TAG:-${APP_TAG}}" + SERVICES_CSV="${SMARTFLOW_RESTART_SERVICES:-${SERVICES_CSV}}" + app_bundle_path="${bundle_dir}/smartflow-app-${APP_TAG}.tar" + infra_bundle_path="${bundle_dir}/smartflow-infra-${APP_TAG}.tar" + rm -f "${app_bundle_path}" fi -if [[ "${SKIP_FRONTEND}" -eq 0 ]]; then - echo "==> Build frontend image ${frontend_ref}" - docker build --platform linux/amd64 -f "${repo_root}/frontend/Dockerfile" -t "${frontend_ref}" "${repo_root}/frontend" +declare -a services=() +if [[ -n "${SERVICES_CSV}" ]]; then + IFS=',' read -r -a services <<< "${SERVICES_CSV}" +elif [[ -z "${PLAN_FILE}" ]]; then + # 1. 手工执行且没有传 plan 时,默认构建所有应用服务,保持旧脚本“一键全量打包”的可用性。 + # 2. 这里改为服务级镜像全量,而不是 backend-suite,便于后续逐步淘汰单体后端镜像。 + if [[ "${SKIP_BACKEND}" -eq 0 ]]; then + services+=("${SMARTFLOW_BACKEND_SERVICES[@]}") + fi + if [[ "${SKIP_FRONTEND}" -eq 0 ]]; then + services+=("frontend") + fi fi +image_ref_for_service() { + local service="$1" + local image_env + local image_ref + + image_env="$(smartflow_image_env_for_service "${service}")" + local -n image_value="${image_env}" + image_ref="${image_value:-}" + if [[ -z "${image_ref}" ]]; then + image_ref="$(smartflow_default_image_for_service "${service}" "${APP_TAG}")" + fi + + printf '%s\n' "${image_ref}" +} + declare -a app_images=() -if [[ "${SKIP_BACKEND}" -eq 0 ]]; then - app_images+=("${backend_ref}") -fi -if [[ "${SKIP_FRONTEND}" -eq 0 ]]; then - app_images+=("${frontend_ref}") -fi +for service in "${services[@]}"; do + if [[ -z "${service}" ]]; then + continue + fi + + if [[ "${service}" == "frontend" ]]; then + if [[ "${SKIP_FRONTEND}" -eq 1 ]]; then + continue + fi + + image_ref="$(image_ref_for_service "${service}")" + echo "==> Build frontend image ${image_ref}" + docker build --platform linux/amd64 -f "${repo_root}/frontend/Dockerfile" -t "${image_ref}" "${repo_root}/frontend" + app_images+=("${image_ref}") + continue + fi + + if [[ "${SKIP_BACKEND}" -eq 1 ]]; then + continue + fi + + if ! smartflow_is_backend_service "${service}"; then + echo "unknown release service: ${service}" >&2 + exit 65 + fi + + image_ref="$(image_ref_for_service "${service}")" + echo "==> Build backend service image ${image_ref}" + docker build \ + --platform linux/amd64 \ + --target runtime-service \ + --build-arg "SERVICE=${service}" \ + -f "${repo_root}/backend/Dockerfile" \ + -t "${image_ref}" \ + "${repo_root}/backend" + app_images+=("${image_ref}") +done if [[ "${#app_images[@]}" -gt 0 ]]; then - rm -f "${app_bundle_path}" echo "==> Export app bundle to ${app_bundle_path}" docker save -o "${app_bundle_path}" "${app_images[@]}" else - echo "==> Skip app bundle export because backend/frontend are both skipped" + echo "==> Skip app bundle export because no application image is selected" fi if [[ "${INCLUDE_INFRA}" -eq 0 ]]; then diff --git a/deploy/impact-rules.ps1 b/deploy/impact-rules.ps1 new file mode 100644 index 0000000..2b6309c --- /dev/null +++ b/deploy/impact-rules.ps1 @@ -0,0 +1,151 @@ +param( + [string]$BaseRef = "", + [string]$HeadRef = "HEAD", + [string]$OutputFile = "" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest +. (Join-Path $PSScriptRoot "service-catalog.ps1") + +$repoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $repoRoot + +function Test-GitRef { + param([string]$Ref) + + if ([string]::IsNullOrWhiteSpace($Ref)) { + return $false + } + + & git rev-parse --verify --quiet "$Ref^{commit}" | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Add-SelectedService { + param( + [System.Collections.Generic.List[string]]$Services, + [string]$Service + ) + + if (-not $Services.Contains($Service)) { + $Services.Add($Service) + } +} + +if (-not (Test-GitRef -Ref $HeadRef)) { + throw ("head ref not found: {0}" -f $HeadRef) +} + +$appTag = (& git rev-parse --short=12 $HeadRef).Trim() +$selectedServices = [System.Collections.Generic.List[string]]::new() +$frontendChanged = $false +$fullBackend = $false + +if (Test-GitRef -Ref $BaseRef) { + $changedFiles = @(& git diff --name-only $BaseRef $HeadRef) +} else { + $changedFiles = @() + $frontendChanged = $true + $fullBackend = $true +} + +foreach ($file in $changedFiles) { + switch -Wildcard ($file) { + "README.md" { continue } + "docs/*" { continue } + "frontend/*" { $frontendChanged = $true; continue } + "deploy/nginx/*" { $frontendChanged = $true; continue } + "deploy/docker-pack.*" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/docker-load.sh" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/stage-release.*" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/project-release.sh" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/project-rollback.sh" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/impact-rules.*" { $frontendChanged = $true; $fullBackend = $true; continue } + "deploy/service-catalog.*" { $frontendChanged = $true; $fullBackend = $true; continue } + "docker-compose.full.yml" { $frontendChanged = $true; $fullBackend = $true; continue } + "frontend/nginx.conf" { $frontendChanged = $true; $fullBackend = $true; continue } + ".env.full.example" { $frontendChanged = $true; $fullBackend = $true; continue } + "backend/Dockerfile" { $fullBackend = $true; continue } + "backend/config.docker.yaml" { $fullBackend = $true; continue } + "backend/shared/*" { $fullBackend = $true; continue } + "backend/client/*" { $fullBackend = $true; continue } + "backend/gateway/*" { Add-SelectedService -Services $selectedServices -Service "api"; continue } + "backend/cmd/api/*" { Add-SelectedService -Services $selectedServices -Service "api"; continue } + "backend/cmd/userauth/*" { Add-SelectedService -Services $selectedServices -Service "userauth"; continue } + "backend/services/userauth/*" { Add-SelectedService -Services $selectedServices -Service "userauth"; continue } + "backend/cmd/notification/*" { Add-SelectedService -Services $selectedServices -Service "notification"; continue } + "backend/services/notification/*" { Add-SelectedService -Services $selectedServices -Service "notification"; continue } + "backend/cmd/active-scheduler/*" { Add-SelectedService -Services $selectedServices -Service "active-scheduler"; continue } + "backend/services/active_scheduler/*" { Add-SelectedService -Services $selectedServices -Service "active-scheduler"; continue } + "backend/cmd/schedule/*" { Add-SelectedService -Services $selectedServices -Service "schedule"; continue } + "backend/services/schedule/*" { Add-SelectedService -Services $selectedServices -Service "schedule"; continue } + "backend/cmd/task/*" { Add-SelectedService -Services $selectedServices -Service "task"; continue } + "backend/services/task/*" { Add-SelectedService -Services $selectedServices -Service "task"; continue } + "backend/cmd/task-class/*" { Add-SelectedService -Services $selectedServices -Service "task-class"; continue } + "backend/services/task_class/*" { Add-SelectedService -Services $selectedServices -Service "task-class"; continue } + "backend/cmd/course/*" { Add-SelectedService -Services $selectedServices -Service "course"; continue } + "backend/services/course/*" { Add-SelectedService -Services $selectedServices -Service "course"; continue } + "backend/cmd/memory/*" { Add-SelectedService -Services $selectedServices -Service "memory"; continue } + "backend/services/memory/*" { Add-SelectedService -Services $selectedServices -Service "memory"; continue } + "backend/cmd/agent/*" { Add-SelectedService -Services $selectedServices -Service "agent"; continue } + "backend/services/agent/*" { Add-SelectedService -Services $selectedServices -Service "agent"; continue } + "backend/cmd/taskclassforum/*" { Add-SelectedService -Services $selectedServices -Service "taskclassforum"; continue } + "backend/services/taskclassforum/*" { Add-SelectedService -Services $selectedServices -Service "taskclassforum"; continue } + "backend/cmd/tokenstore/*" { Add-SelectedService -Services $selectedServices -Service "tokenstore"; continue } + "backend/services/tokenstore/*" { Add-SelectedService -Services $selectedServices -Service "tokenstore"; continue } + "backend/cmd/llm/*" { Add-SelectedService -Services $selectedServices -Service "llm"; continue } + "backend/services/llm/*" { Add-SelectedService -Services $selectedServices -Service "llm"; continue } + "backend/*" { $fullBackend = $true; continue } + } +} + +if ($fullBackend) { + $selectedServices.Clear() + foreach ($service in Get-SmartFlowBackendServices) { + $selectedServices.Add($service) + } +} + +if ($frontendChanged) { + Add-SelectedService -Services $selectedServices -Service "frontend" +} + +$buildBackend = 0 +$buildFrontend = 0 +foreach ($service in $selectedServices) { + if ($service -eq "frontend") { + $buildFrontend = 1 + } else { + $buildBackend = 1 + } +} + +$noop = if ($selectedServices.Count -eq 0) { 1 } else { 0 } +$restartCsv = [string]::Join(",", $selectedServices.ToArray()) + +$lines = [System.Collections.Generic.List[string]]::new() +$lines.Add("SMARTFLOW_APP_TAG=$appTag") +$lines.Add("SMARTFLOW_NOOP=$noop") +$lines.Add("SMARTFLOW_BUILD_BACKEND=$buildBackend") +$lines.Add("SMARTFLOW_BUILD_FRONTEND=$buildFrontend") +$lines.Add("SMARTFLOW_RESTART_SERVICES=$restartCsv") + +foreach ($service in $selectedServices) { + $imageEnv = Get-SmartFlowImageEnvForService -Service $service + $imageRef = Get-SmartFlowDefaultImageForService -Service $service -AppTag $appTag + $lines.Add(("{0}={1}" -f $imageEnv, $imageRef)) +} + +$content = [string]::Join("`n", $lines.ToArray()) +if (-not [string]::IsNullOrWhiteSpace($OutputFile)) { + $parent = Split-Path -Parent $OutputFile + if (-not [string]::IsNullOrWhiteSpace($parent)) { + New-Item -ItemType Directory -Force -Path $parent | Out-Null + } + $outputPath = if ([System.IO.Path]::IsPathRooted($OutputFile)) { $OutputFile } else { Join-Path (Get-Location) $OutputFile } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::WriteAllText($outputPath, $content, $utf8NoBom) +} else { + Write-Output $content +} diff --git a/deploy/impact-rules.sh b/deploy/impact-rules.sh index cb0602b..b67431d 100755 --- a/deploy/impact-rules.sh +++ b/deploy/impact-rules.sh @@ -8,7 +8,8 @@ output_file="${3:-}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${repo_root}" -backend_services=(userauth notification active-scheduler schedule task task-class course memory agent taskclassforum tokenstore llm api) +source "${repo_root}/deploy/service-catalog.sh" + selected_services=() frontend_changed=0 full_backend=0 @@ -50,7 +51,7 @@ for file in "${changed_files[@]}"; do frontend/*|deploy/nginx/*) frontend_changed=1 ;; - deploy/docker-pack.*|deploy/docker-load.sh|deploy/stage-release.sh|deploy/project-release.sh|deploy/project-rollback.sh|deploy/impact-rules.sh) + deploy/docker-pack.*|deploy/docker-load.sh|deploy/stage-release.*|deploy/project-release.sh|deploy/project-rollback.sh|deploy/impact-rules.*|deploy/service-catalog.*) frontend_changed=1 full_backend=1 ;; @@ -107,7 +108,7 @@ for file in "${changed_files[@]}"; do done if [[ "${full_backend}" -eq 1 ]]; then - selected_services=("${backend_services[@]}") + selected_services=("${SMARTFLOW_BACKEND_SERVICES[@]}") fi if [[ "${frontend_changed}" -eq 1 ]]; then @@ -140,12 +141,16 @@ SMARTFLOW_APP_TAG=${app_tag} SMARTFLOW_NOOP=${noop} SMARTFLOW_BUILD_BACKEND=${build_backend} SMARTFLOW_BUILD_FRONTEND=${build_frontend} -SMARTFLOW_BACKEND_IMAGE=smartflow/backend-suite:${app_tag} -SMARTFLOW_FRONTEND_IMAGE=smartflow/frontend:${app_tag} SMARTFLOW_RESTART_SERVICES=${restart_csv} EOF ) +for service in "${selected_services[@]}"; do + image_env="$(smartflow_image_env_for_service "${service}")" + image_ref="$(smartflow_default_image_for_service "${service}" "${app_tag}")" + content+=$'\n'"${image_env}=${image_ref}" +done + if [[ -n "${output_file}" ]]; then printf '%s\n' "${content}" > "${output_file}" else diff --git a/deploy/project-release.sh b/deploy/project-release.sh index 26fdd64..7a372b1 100755 --- a/deploy/project-release.sh +++ b/deploy/project-release.sh @@ -11,6 +11,8 @@ release_id="${SMARTFLOW_RELEASE_ID:?SMARTFLOW_RELEASE_ID is required}" plan_file="${release_dir}/deploy/release-plan.env" runtime_env="${runtime_dir}/.env" +source "${release_dir}/deploy/service-catalog.sh" + if [[ ! -f "${plan_file}" ]]; then echo "release plan not found: ${plan_file}" >&2 exit 66 @@ -61,14 +63,6 @@ if compgen -G "${release_dir}/.docker-bundles/*.tar" >/dev/null 2>&1; then bash "${release_dir}/deploy/docker-load.sh" "${release_dir}/.docker-bundles" fi -if [[ "${SMARTFLOW_BUILD_BACKEND:-0}" == "1" ]]; then - set_env_var "SMARTFLOW_BACKEND_IMAGE" "${SMARTFLOW_BACKEND_IMAGE}" "${runtime_env}" -fi - -if [[ "${SMARTFLOW_BUILD_FRONTEND:-0}" == "1" ]]; then - set_env_var "SMARTFLOW_FRONTEND_IMAGE" "${SMARTFLOW_FRONTEND_IMAGE}" "${runtime_env}" -fi - services=() IFS=',' read -r -a raw_services <<< "${SMARTFLOW_RESTART_SERVICES:-}" for service in "${raw_services[@]}"; do @@ -82,6 +76,19 @@ if [[ "${#services[@]}" -eq 0 ]]; then exit 0 fi +# 1. release-plan.env 是构建机生成的单一事实源,部署机只按服务名读取对应镜像变量。 +# 2. 某个服务缺少镜像变量时直接失败,避免 compose 沿用旧镜像造成“发布成功但代码未更新”。 +# 3. .env 更新发生在 docker load 之后;如果镜像包无法加载,不会提前切换运行时镜像引用。 +for service in "${services[@]}"; do + image_env="$(smartflow_image_env_for_service "${service}")" + local_image_ref="${!image_env:-}" + if [[ -z "${local_image_ref}" ]]; then + echo "image ref not found in release plan: ${image_env}" >&2 + exit 68 + fi + set_env_var "${image_env}" "${local_image_ref}" "${runtime_env}" +done + # 1. 使用 --no-deps 只重建命中的服务,避免后端小改动把整套依赖链一起拉起来。 # 2. 如果后续服务新增或删减,只要 release-plan.env 给出的服务名同步更新,这里无需改脚本。 # 3. 失败时直接退出,由上层薄封装决定是否切回旧 release。 diff --git a/deploy/service-catalog.ps1 b/deploy/service-catalog.ps1 new file mode 100644 index 0000000..d0230dc --- /dev/null +++ b/deploy/service-catalog.ps1 @@ -0,0 +1,85 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +if ([string]::IsNullOrWhiteSpace($env:SMARTFLOW_SERVICE_CATALOG_FILE)) { + $script:SmartFlowCatalogFile = Join-Path $PSScriptRoot "service-catalog.txt" +} +if (-not [string]::IsNullOrWhiteSpace($env:SMARTFLOW_SERVICE_CATALOG_FILE)) { + $script:SmartFlowCatalogFile = $env:SMARTFLOW_SERVICE_CATALOG_FILE +} + +function Get-SmartFlowServiceCatalog { + if (-not (Test-Path -LiteralPath $script:SmartFlowCatalogFile)) { + throw ("service catalog not found: {0}" -f $script:SmartFlowCatalogFile) + } + + $items = @() + foreach ($line in Get-Content -LiteralPath $script:SmartFlowCatalogFile -Encoding UTF8) { + if ([string]::IsNullOrWhiteSpace($line) -or $line.TrimStart().StartsWith("#")) { + continue + } + + $parts = $line.Split("|") + if ($parts.Count -ne 4) { + throw ("invalid service catalog line: {0}" -f $line) + } + + $items += [pscustomobject]@{ + Service = $parts[0].Trim() + ImageEnv = $parts[1].Trim() + ImageRepo = $parts[2].Trim() + Kind = $parts[3].Trim() + } + } + + return $items +} + +function Get-SmartFlowBackendServices { + return @(Get-SmartFlowServiceCatalog | Where-Object { $_.Kind -eq "backend" } | ForEach-Object { $_.Service }) +} + +function Get-SmartFlowServiceCatalogItem { + param( + [Parameter(Mandatory = $true)] + [string]$Service + ) + + $item = Get-SmartFlowServiceCatalog | Where-Object { $_.Service -eq $Service } | Select-Object -First 1 + if ($null -eq $item) { + throw ("unknown release service: {0}" -f $Service) + } + + return $item +} + +function Test-SmartFlowBackendService { + param( + [Parameter(Mandatory = $true)] + [string]$Service + ) + + return ((Get-SmartFlowServiceCatalogItem -Service $Service).Kind -eq "backend") +} + +function Get-SmartFlowImageEnvForService { + param( + [Parameter(Mandatory = $true)] + [string]$Service + ) + + return (Get-SmartFlowServiceCatalogItem -Service $Service).ImageEnv +} + +function Get-SmartFlowDefaultImageForService { + param( + [Parameter(Mandatory = $true)] + [string]$Service, + + [Parameter(Mandatory = $true)] + [string]$AppTag + ) + + $item = Get-SmartFlowServiceCatalogItem -Service $Service + return ("{0}:{1}" -f $item.ImageRepo, $AppTag) +} diff --git a/deploy/service-catalog.sh b/deploy/service-catalog.sh new file mode 100644 index 0000000..53b99b7 --- /dev/null +++ b/deploy/service-catalog.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 1. 发布脚本共享同一份服务清单,避免影响计算、镜像构建、部署更新三处各自维护服务名。 +# 2. 这里只把文本清单加载成 Bash 可用的数据结构;不负责判断某次发布要构建哪些服务。 +# 3. 新增或删除后端服务时,优先改 service-catalog.txt,再同步 compose 服务定义,避免脚本之间出现漂移。 +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +catalog_file="${SMARTFLOW_SERVICE_CATALOG_FILE:-${script_dir}/service-catalog.txt}" + +if [[ ! -f "${catalog_file}" ]]; then + echo "service catalog not found: ${catalog_file}" >&2 + exit 66 +fi + +SMARTFLOW_BACKEND_SERVICES=() +declare -Ag SMARTFLOW_SERVICE_IMAGE_ENVS=() +declare -Ag SMARTFLOW_SERVICE_IMAGE_REPOS=() +declare -Ag SMARTFLOW_SERVICE_KINDS=() + +while IFS='|' read -r service image_env image_repo kind; do + if [[ -z "${service}" || "${service}" == \#* ]]; then + continue + fi + + SMARTFLOW_SERVICE_IMAGE_ENVS["${service}"]="${image_env}" + SMARTFLOW_SERVICE_IMAGE_REPOS["${service}"]="${image_repo}" + SMARTFLOW_SERVICE_KINDS["${service}"]="${kind}" + + if [[ "${kind}" == "backend" ]]; then + SMARTFLOW_BACKEND_SERVICES+=("${service}") + fi +done < "${catalog_file}" + +# smartflow_is_backend_service 负责判断服务是否属于后端发布粒度。 +# 输入:compose 服务名。 +# 输出:命中返回 0,未命中返回 1;不负责校验 frontend 这类非后端服务。 +smartflow_is_backend_service() { + local service="$1" + [[ "${SMARTFLOW_SERVICE_KINDS[${service}]-}" == "backend" ]] +} + +# smartflow_image_env_for_service 负责把 compose 服务名映射成运行时 .env 里的镜像变量。 +# 输入:compose 服务名,例如 task-class。 +# 输出:变量名,例如 SMARTFLOW_IMAGE_TASK_CLASS;未知服务直接失败,避免发布脚本静默写错键。 +smartflow_image_env_for_service() { + local service="$1" + local image_env="${SMARTFLOW_SERVICE_IMAGE_ENVS[${service}]-}" + + if [[ -z "${image_env}" ]]; then + echo "unknown service: ${service}" >&2 + return 65 + fi + + echo "${image_env}" +} + +# smartflow_default_image_for_service 负责生成服务级镜像的默认引用。 +# 输入:compose 服务名、应用 tag。 +# 输出:smartflow/:,frontend 保持 smartflow/frontend:。 +smartflow_default_image_for_service() { + local service="$1" + local app_tag="$2" + local image_repo="${SMARTFLOW_SERVICE_IMAGE_REPOS[${service}]-}" + + if [[ -z "${image_repo}" ]]; then + echo "unknown service: ${service}" >&2 + return 65 + fi + + echo "${image_repo}:${app_tag}" +} diff --git a/deploy/service-catalog.txt b/deploy/service-catalog.txt new file mode 100644 index 0000000..20bb76c --- /dev/null +++ b/deploy/service-catalog.txt @@ -0,0 +1,15 @@ +# service|image_env|image_repo|kind +userauth|SMARTFLOW_IMAGE_USERAUTH|smartflow/userauth|backend +notification|SMARTFLOW_IMAGE_NOTIFICATION|smartflow/notification|backend +active-scheduler|SMARTFLOW_IMAGE_ACTIVE_SCHEDULER|smartflow/active-scheduler|backend +schedule|SMARTFLOW_IMAGE_SCHEDULE|smartflow/schedule|backend +task|SMARTFLOW_IMAGE_TASK|smartflow/task|backend +task-class|SMARTFLOW_IMAGE_TASK_CLASS|smartflow/task-class|backend +course|SMARTFLOW_IMAGE_COURSE|smartflow/course|backend +memory|SMARTFLOW_IMAGE_MEMORY|smartflow/memory|backend +agent|SMARTFLOW_IMAGE_AGENT|smartflow/agent|backend +taskclassforum|SMARTFLOW_IMAGE_TASKCLASSFORUM|smartflow/taskclassforum|backend +tokenstore|SMARTFLOW_IMAGE_TOKENSTORE|smartflow/tokenstore|backend +llm|SMARTFLOW_IMAGE_LLM|smartflow/llm|backend +api|SMARTFLOW_IMAGE_API|smartflow/api|backend +frontend|SMARTFLOW_IMAGE_FRONTEND|smartflow/frontend|frontend diff --git a/deploy/stage-release.ps1 b/deploy/stage-release.ps1 new file mode 100644 index 0000000..efa8d04 --- /dev/null +++ b/deploy/stage-release.ps1 @@ -0,0 +1,46 @@ +param( + [Parameter(Mandatory = $true)] + [string]$ReleaseDir, + + [Parameter(Mandatory = $true)] + [string]$PlanFile, + + [string]$BundleDir = ".docker-bundles" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$repoRoot = Split-Path -Parent $PSScriptRoot +$releaseAbs = Join-Path $repoRoot $ReleaseDir +$bundleAbs = Join-Path $repoRoot $BundleDir + +if (-not (Test-Path -LiteralPath $PlanFile)) { + throw ("release plan not found: {0}" -f $PlanFile) +} + +if (Test-Path -LiteralPath $releaseAbs) { + Remove-Item -LiteralPath $releaseAbs -Recurse -Force +} + +New-Item -ItemType Directory -Force -Path (Join-Path $releaseAbs "deploy\nginx") | Out-Null +New-Item -ItemType Directory -Force -Path (Join-Path $releaseAbs "deploy\certs") | Out-Null +New-Item -ItemType Directory -Force -Path (Join-Path $releaseAbs ".docker-bundles") | Out-Null + +Copy-Item -LiteralPath (Join-Path $repoRoot "docker-compose.full.yml") -Destination (Join-Path $releaseAbs "docker-compose.full.yml") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\docker-load.sh") -Destination (Join-Path $releaseAbs "deploy\docker-load.sh") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\project-release.sh") -Destination (Join-Path $releaseAbs "deploy\project-release.sh") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\project-rollback.sh") -Destination (Join-Path $releaseAbs "deploy\project-rollback.sh") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\impact-rules.sh") -Destination (Join-Path $releaseAbs "deploy\impact-rules.sh") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\service-catalog.sh") -Destination (Join-Path $releaseAbs "deploy\service-catalog.sh") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\service-catalog.txt") -Destination (Join-Path $releaseAbs "deploy\service-catalog.txt") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\nginx\default.conf") -Destination (Join-Path $releaseAbs "deploy\nginx\default.conf") +Copy-Item -LiteralPath (Join-Path $repoRoot "deploy\certs\README.md") -Destination (Join-Path $releaseAbs "deploy\certs\README.md") +Copy-Item -LiteralPath $PlanFile -Destination (Join-Path $releaseAbs "deploy\release-plan.env") + +if (Test-Path -LiteralPath $bundleAbs) { + $bundles = Get-ChildItem -LiteralPath $bundleAbs -Filter "*.tar" -File + foreach ($bundle in $bundles) { + Copy-Item -LiteralPath $bundle.FullName -Destination (Join-Path $releaseAbs ".docker-bundles") + } +} diff --git a/deploy/stage-release.sh b/deploy/stage-release.sh index 6977736..e8f03db 100755 --- a/deploy/stage-release.sh +++ b/deploy/stage-release.sh @@ -43,6 +43,8 @@ cp "${repo_root}/deploy/docker-load.sh" "${release_abs}/deploy/docker-load.sh" cp "${repo_root}/deploy/project-release.sh" "${release_abs}/deploy/project-release.sh" cp "${repo_root}/deploy/project-rollback.sh" "${release_abs}/deploy/project-rollback.sh" cp "${repo_root}/deploy/impact-rules.sh" "${release_abs}/deploy/impact-rules.sh" +cp "${repo_root}/deploy/service-catalog.sh" "${release_abs}/deploy/service-catalog.sh" +cp "${repo_root}/deploy/service-catalog.txt" "${release_abs}/deploy/service-catalog.txt" cp "${repo_root}/deploy/nginx/default.conf" "${release_abs}/deploy/nginx/default.conf" cp "${repo_root}/deploy/certs/README.md" "${release_abs}/deploy/certs/README.md" cp "${plan_file}" "${release_abs}/deploy/release-plan.env" diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 7209685..487c10d 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -1,7 +1,6 @@ name: smartflow-full x-backend-common: &backend-common - image: ${SMARTFLOW_BACKEND_IMAGE:-smartflow/backend-suite:latest} restart: unless-stopped working_dir: /app/backend environment: @@ -194,54 +193,67 @@ services: userauth: <<: *backend-common + image: ${SMARTFLOW_IMAGE_USERAUTH:-smartflow/userauth:latest} command: ["/app/bin/userauth"] notification: <<: *backend-common + image: ${SMARTFLOW_IMAGE_NOTIFICATION:-smartflow/notification:latest} command: ["/app/bin/notification"] active-scheduler: <<: *backend-common + image: ${SMARTFLOW_IMAGE_ACTIVE_SCHEDULER:-smartflow/active-scheduler:latest} command: ["/app/bin/active-scheduler"] schedule: <<: *backend-common + image: ${SMARTFLOW_IMAGE_SCHEDULE:-smartflow/schedule:latest} command: ["/app/bin/schedule"] task: <<: *backend-common + image: ${SMARTFLOW_IMAGE_TASK:-smartflow/task:latest} command: ["/app/bin/task"] task-class: <<: *backend-common + image: ${SMARTFLOW_IMAGE_TASK_CLASS:-smartflow/task-class:latest} command: ["/app/bin/task-class"] course: <<: *backend-common + image: ${SMARTFLOW_IMAGE_COURSE:-smartflow/course:latest} command: ["/app/bin/course"] memory: <<: *backend-common + image: ${SMARTFLOW_IMAGE_MEMORY:-smartflow/memory:latest} command: ["/app/bin/memory"] agent: <<: *backend-common + image: ${SMARTFLOW_IMAGE_AGENT:-smartflow/agent:latest} command: ["/app/bin/agent"] taskclassforum: <<: *backend-common + image: ${SMARTFLOW_IMAGE_TASKCLASSFORUM:-smartflow/taskclassforum:latest} command: ["/app/bin/taskclassforum"] tokenstore: <<: *backend-common + image: ${SMARTFLOW_IMAGE_TOKENSTORE:-smartflow/tokenstore:latest} command: ["/app/bin/tokenstore"] llm: <<: *backend-common + image: ${SMARTFLOW_IMAGE_LLM:-smartflow/llm:latest} command: ["/app/bin/llm"] api: <<: *backend-common + image: ${SMARTFLOW_IMAGE_API:-smartflow/api:latest} command: ["/app/bin/api"] ports: - "${SMARTFLOW_API_PORT:-8080}:8080" @@ -286,7 +298,7 @@ services: condition: service_started frontend: - image: ${SMARTFLOW_FRONTEND_IMAGE:-smartflow/frontend:latest} + image: ${SMARTFLOW_IMAGE_FRONTEND:-smartflow/frontend:latest} restart: unless-stopped ports: - "${SMARTFLOW_FRONTEND_PORT:-80}:80"