おうちk8sでMisskeyとNowPlayingを動かし始めたよ · usbharu - blogで動かし始めたk3sクラスタですが、SSDの物理故障により停止、そのまま廃止となりました。その後は各サーバーでdocker composeを動かしていましたが、やはりYAMLを編集するだけで複数サーバーのコンテナを管理できるのは楽です。
そもそも欲しいものとは#
そもそも、私がほしいのはコンテナオーケストレーションではなく、複数サーバーのコンテナのGitOpsです。また、全てを宣言的に管理したいわけでもなく、自動修正もそこまで欲しいわけではありません。別にDocker Composeである必要はありませんが、慣れているツールがそのまま使えるならそっちの方が良いでしょう。
Komodo#
https://github.com/moghtech/komodo
Komodoはサーバーに配置しているdocker composeファイルをGit管理してくれるもので、ArgoCDと同じようにGitサーバーを定期的にポーリングまたはWebhookにより監視し、変更があった場合反映してくれるものです。

サーバーのメトリクスを見れたり、アラートを送ったり、コンテナイメージのビルドもできるようですが、その辺はGrafanaやCIに任せています。Giteaと一緒に建てたらお手軽CI/CD+デプロイ先環境が整う気がしますね。
KomodoはCoreとperipheryに分かれており、Coreが管理役、peripheryが実際にサーバーで変更を適用します。DBにはMongoを使います。
Komodoでは一つの管理単位のことをStackと呼び(ArgoCDで言うところのApplication)、このStack自体の管理はTomlで行います。
# Apply bootstrap/sync.toml separately; a running sync cannot update itself.
[[stack]]
name = "ocms"
description = "OCMS on host PostgreSQL and Cloudflare R2; GHCR release images and tracked database migrations."
tags = ["usbharu-lab"]
# The procedure deploys stacks in dependency order after Sync updates definitions.
deploy = false
[stack.config]
server = "ocms-ubuntu"
repo = "usbharu/usbharu-lab"
branch = "komodo"
git_provider = "github.com"
files_on_host = false
clone_path = "/etc/komodo/gitops/ocms"
run_directory = "stacks/ocms"
file_paths = ["compose.yaml"]
project_name = "ocms"
ignore_services = ["migrate"]
env_file_path = ".komodo.env"
additional_env_files = [{ path = "/etc/komodo/stacks/management/ocms.env", track = false }]
environment = """
R2_ACCESS_KEY_ID = [[OCMS_R2_ACCESS_KEY_ID]]
R2_SECRET_ACCESS_KEY = [[OCMS_R2_SECRET_ACCESS_KEY]]
"""
auto_pull = false
run_build = false
poll_for_updates = false
auto_update = false
destroy_before_deploy = false
webhook_enabled = false
extra_args = ["--wait", "--wait-timeout", "300"]
[[stack]]
name = "ocms-management"
description = "Komodo, Pocket ID and Traefik; credentials and state stay on the host."
tags = ["usbharu-lab"]
deploy = false
[stack.config]
server = "ocms-ubuntu"
repo = "usbharu/usbharu-lab"
branch = "komodo"
git_provider = "github.com"
files_on_host = false
clone_path = "/etc/komodo/gitops/management"
run_directory = "stacks/management"
file_paths = ["compose.yaml"]
config_files = [{ path = "traefik/routes.yaml", services = ["traefik"], requires = "Restart" }]
project_name = "ocms-management"
env_file_path = ".komodo.env"
additional_env_files = [{ path = "/etc/komodo/stacks/management/.env", track = false }]
auto_pull = false
run_build = false
poll_for_updates = false
auto_update = false
destroy_before_deploy = false
webhook_enabled = false
extra_args = ["--wait", "--wait-timeout", "300"]
[[stack]]
name = "ocms-observability"
description = "Grafana Alloy: Ubuntu/Docker/OCMS metrics to PVE LGTM; Komodo/Docker logs and OCMS logs/traces to Grafana Cloud."
tags = ["usbharu-lab"]
deploy = false
[stack.config]
server = "ocms-ubuntu"
repo = "usbharu/usbharu-lab"
branch = "komodo"
git_provider = "github.com"
files_on_host = false
clone_path = "/etc/komodo/gitops/observability"
run_directory = "stacks/observability"
file_paths = ["compose.yaml"]
config_files = [{ path = "alloy/config.alloy", services = ["alloy"], requires = "Restart" }]
project_name = "ocms-observability"
env_file_path = ".komodo.env"
additional_env_files = [{ path = "/etc/komodo/secrets/observability/cloud.env", track = false }]
auto_pull = false
run_build = false
poll_for_updates = false
auto_update = false
destroy_before_deploy = false
webhook_enabled = false
extra_args = ["--wait", "--wait-timeout", "300"]
[[procedure]]
name = "usbharu-lab-gitops"
description = "GitHub push to komodo branch, plus five-minute reconciliation."
tags = ["usbharu-lab"]
[procedure.config]
schedule_format = "Cron"
schedule = "0 */5 * * * *"
schedule_timezone = "Asia/Tokyo"
schedule_enabled = true
schedule_alert = false
failure_alert = true
webhook_enabled = true
[[procedure.config.stage]]
name = "Sync resource definitions"
executions = [{ execution.type = "RunSync", execution.params.sync = "usbharu-lab" }]
[[procedure.config.stage]]
name = "Deploy observability if changed"
executions = [{ execution.type = "DeployStackIfChanged", execution.params.stack = "ocms-observability" }]
[[procedure.config.stage]]
name = "Deploy OCMS if changed"
executions = [{ execution.type = "DeployStackIfChanged", execution.params.stack = "ocms" }]
[[procedure.config.stage]]
name = "Deploy management if changed"
executions = [{ execution.type = "DeployStackIfChanged", execution.params.stack = "ocms-management" }]所感#
k8sほど複雑でなく、WebUIもあり、そこそこの設定項目があるので使いやすいなと感じています。WebUIが要らない人はcompose-cdとかを使えばいいかなぁと言った感じです。前回設定したPocket IDによるSSOを設定しているのですが、これがやはり便利です。

