updates README, housekeeping hosts groups

README.md

@@ -1,18 +1,124 @@
 # cit Ansible Infrastructure
 
+This repository contains the Ansible based infrastructure setup for cit services running on Hetzner virtual machines. It covers initial host bootstrapping, common system configuration, Docker and Docker Compose based application deployment, and ongoing maintenance.
+
+Parts of this setup are inspired by and derived from existing open source projects. See the Open Source Acknowledgements section at the end.
+
+## Requirements
+
+* Ansible installed locally
+* SSH access to Hetzner hosts defined in `inventories/hetzner/hosts.yaml`
+* Host specific variables in `inventories/hetzner/host_vars/`
+* Debian based target systems
+
+## Structure
+
+* `inventories/hetzner/hosts.yaml`
+  * Host definitions and group assignments
+* `inventories/hetzner/host_vars/`
+  * Host specific configuration
+  * Includes Docker Compose projects and service configuration
+* `docker_compose_applications/`
+  * Docker Compose services, one "stack" per folder
+
+## Common Playbooks
+
+The following playbooks cover the most common workflows. Replace `HOSTNAME` with the inventory hostname.
+
+### Init VM
+
+First bootstrap of a freshly created VM. This is usually run once and requires root access.
+
+```sh
+ansible-playbook -e ansible_user=root -i inventories/hetzner playbooks/00-init_hetzer_vm.yaml
+```
+
+### Everything
+
+Maintenance and full (re-)deployment.
+
+```sh
+ansible-playbook -i inventories/hetzner playbooks/01-everything.yaml
+```
+
+### Maintenance
+
+Regular system updates and patching.
+
+```sh
+ansible-playbook -i inventories/hetzner playbooks/02-maintenance.yaml
+```
+
+### "compose up"
+
+Deploy or refresh Docker Compose applications after configuration changes or when adding new services.
+
+```sh
+ansible-playbook -i inventories/hetzner playbooks/04-compose-up.yml
+```
+
+### Rebuild Caddy
+
+Rebuild and reload Caddy only. Useful when reverse proxy configuration changed and full service restarts should be avoided.
+
+```sh
+ansible-playbook -i inventories/hetzner playbooks/05-rebuild-caddy.yml
+```
+
+#### Notes
+
+* Some services such as Stirling PDF have a slow startup time. When only reverse proxy configuration changes are required, prefer the Caddy rebuild playbook.
+
 ## Adding a new Hetzner VM
 
-1. Add the VM to `inventories/hetzner/hosts.yaml`  
+1. Add the host to `inventories/hetzner/hosts.yaml`
-   Set the correct `ansible_host` and `ansible_user`, add it to the correct Debian groups, but aside from that there's no further configuration needed for now.
+   * Set `ansible_host` and `ansible_user`
-2. Initialize the VM using the following command (replace `HOST_NAME_HERE` with the name in the inventory):
+2. Bootstrap the VM as root
-   ```bash
-   ansible-playbook -e ansible_user=root -i inventories/hetzner -l HOST_NAME_HERE playbooks/init_hetzer_vm.yaml
-   ```
-3. Add the VM to the desired groups and set the desired variables.  
-   Do all the work you need to do (like creating new Ansible roles etc.).
-4. Run `ansible-playbook -i inventories/hetzner playbooks/docker-compose.yml`
 
-## Open Source Software Used
+```bash
+ansible-playbook -e ansible_user=root -i inventories/hetzner -l HOSTNAME playbooks/00-init_hetzer_vm.yaml
+```
 
-Source code was taken from the [CCCHH/ansible-infra repo](https://git.hamburg.ccc.de/CCCHH/ansible-infra), where it was licensed under the MIT license.
+3. Add host variables and group assignments in `inventories/hetzner/host_vars/`
-A copy of the license can be found under [`licenses/ccchh-ansible_mit_license`](licenses/ccchh-ansible_mit_license).
+4. Run the desired playbook, usually the full setup
+
+```bash
+ansible-playbook -i inventories/hetzner -l HOSTNAME playbooks/01-everything.yaml
+```
+
+## Adding a new service
+
+The helper script `add-service.sh` can be used to add new services quickly.
+
+### Clone and register a Git repository
+
+```bash
+./add-service.sh git@github.com:example/repo.git
+```
+
+### Clone with a custom service name
+
+```bash
+./add-service.sh git@github.com:example/repo.git my-service
+```
+
+### Create a local scaffold
+
+Creates a new service directory with a template `compose.yaml` including the shared `caddy_net` network.
+
+```bash
+./add-service.sh my-service
+```
+
+After adding or modifying services, deploy them using the compose up playbook.
+
+## Open Source Acknowledgements
+
+Parts of this repository are based on work from the following project:
+
+* CCCHH ansible infra repository
+
+  * Source: [https://git.hamburg.ccc.de/CCCHH/ansible-infra](https://git.hamburg.ccc.de/CCCHH/ansible-infra)
+  * License: MIT
+
+A copy of the MIT license can be found at `licenses/ccchh-ansible_mit_license`.

add-service.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+APPS_DIR="${ROOT_DIR}/docker_compose_applications"
+INVENTORY_FILE="${ROOT_DIR}/inventories/hetzner/host_vars/cit-docker-host.yaml"
+
+log() {
+  printf '[add-service] %s\n' "$*"
+}
+
+die() {
+  printf '[add-service] ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") <github-ssh-uri|service-name> [service-name]
+
+Examples:
+  $(basename "$0") git@github.com:example/repo.git
+  $(basename "$0") git@github.com:example/repo.git custom-name
+  $(basename "$0") my-new-service
+
+When a GitHub SSH URI is provided, the repo is cloned into docker_compose_applications/
+and the inventory is updated. When plain text is provided, a scaffold folder with a
+compose.yaml template is created and the inventory is updated.
+EOF
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
+}
+
+ensure_paths() {
+  [ -d "$APPS_DIR" ] || mkdir -p "$APPS_DIR"
+  [ -f "$INVENTORY_FILE" ] || die "services list $INVENTORY_FILE not found"
+}
+
+is_github_ssh_uri() {
+  [[ "$1" =~ ^git@github\.com:.+\.git$ ]]
+}
+
+sanitize_name() {
+  local raw="$1"
+  local clean
+  clean="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//; s/-+/-/g')"
+  [ -n "$clean" ] || die "Service name resolved to empty after sanitizing"
+  printf '%s' "$clean"
+}
+
+service_in_inventory() {
+  local name="$1"
+  grep -Eq "^[[:space:]]+- name: ${name}$" "$INVENTORY_FILE"
+}
+
+append_inventory_entry() {
+  local name="$1"
+  local dir_name="$2"
+
+  if service_in_inventory "$name"; then
+    die "Service '${name}' already present in inventory"
+  fi
+
+  {
+    printf '\n  - name: %s\n' "$name"
+    printf '    files_directory: ../docker_compose_applications/%s\n' "$dir_name"
+  } >>"$INVENTORY_FILE"
+
+  log "Added '${name}' to inventory"
+}
+
+derive_name_from_uri() {
+  local uri="$1"
+  local repo="${uri##*/}"
+  repo="${repo%.git}"
+  sanitize_name "$repo"
+}
+
+clone_repo() {
+  local uri="$1"
+  local target_dir="$2"
+
+  [ -e "$target_dir" ] && die "Target directory already exists: $target_dir"
+  require_cmd git
+  log "Cloning ${uri} into ${target_dir}"
+  git clone --depth=1 "$uri" "$target_dir"
+}
+
+create_compose_template() {
+  local dir="$1"
+  local service_name="$2"
+  local compose_path="${dir}/compose.yaml"
+
+  [ -e "$compose_path" ] && die "compose.yaml already exists at ${compose_path}"
+
+  cat >"$compose_path" <<EOF
+services:
+  ${service_name}:
+    image: replace-me
+    container_name: ${service_name}
+    restart: unless-stopped
+    networks:
+      - caddy_net
+
+networks:
+  caddy_net:
+    external: true
+EOF
+  log "Created compose template at ${compose_path}"
+}
+
+create_scaffold() {
+  local dir="$1"
+  local service_name="$2"
+
+  [ -e "$dir" ] && die "Target directory already exists: $dir"
+  mkdir -p "$dir"
+  create_compose_template "$dir" "$service_name"
+}
+
+main() {
+  local input="${1:-}"
+  local override_name="${2:-}"
+
+  [ -n "$input" ] || usage
+  ensure_paths
+
+  if is_github_ssh_uri "$input"; then
+    local service_name="${override_name:-$(derive_name_from_uri "$input")}"
+    local target_dir="${APPS_DIR}/${service_name}"
+    clone_repo "$input" "$target_dir"
+    append_inventory_entry "$service_name" "$service_name"
+    log "Service '${service_name}' ready from GitHub source"
+  else
+    local service_name="${override_name:-$(sanitize_name "$input")}"
+    local target_dir="${APPS_DIR}/${service_name}"
+    create_scaffold "$target_dir" "$service_name"
+    append_inventory_entry "$service_name" "$service_name"
+    log "Service '${service_name}' scaffolded locally"
+  fi
+}
+
+main "$@"

caddy-update.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-ansible-playbook -i inventories/hetzner playbooks/05-update-caddy.yml
+ansible-playbook -i inventories/hetzner ./playbooks/05-rebuild-caddy.yml

inventories/hetzner/hosts.yaml

@@ -4,15 +4,6 @@ all:
       ansible_host: 23.88.36.145
       ansible_user: cit
   children:
-    Production_Hosts:
-      hosts:
-        cit-docker-host:
-    Debian_Hosts:
-      hosts:
-        cit-docker-host:
-    Debian_12_Hosts:
-      hosts:
-        cit-docker-host:
     Docker_Hosts:
       hosts:
         cit-docker-host: