docs(envvars): Automatically update environment files

.pre-commit-config.yaml

@@ -16,6 +16,15 @@ repos:
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
 
+  - repo: local
+    hooks:
+      - id: update-env-docs
+        name: Check Environment Variables Markdown
+        entry: make update-env-docs CHECK=true
+        language: system
+        pass_filenames: false
+        always_run: true
+
   - repo: https://github.com/astral-sh/uv-pre-commit
     rev: 0.10.7
     hooks:

Makefile

@@ -175,4 +175,12 @@ copy-libs:
 	@docker compose cp translator:/app/attribute_pb2_grpc.py translator/
 	@docker compose cp translator:/app/capability_pb2.py translator/
 	@docker compose cp translator:/app/capability_pb2.pyi translator/
-	@docker compose cp translator:/app/capability_pb2_grpc.py translator/
+
+## update-env-docs: update environment variable documentation append CHECK=true to get a diff if not up to date
+.Phony: update-env-docs
+update-env-docs:
+ifeq ($(CHECK),true)
+	@uv run scripts/extract_env_vars.py --check
+else
+	@uv run scripts/extract_env_vars.py
+endif

compose.yml

@@ -1,5 +1,4 @@
 ---
-
 services:
   django:
     build:

config/settings/base.py

@@ -290,6 +290,7 @@
 CORS_URLS_REGEX = r"^/api/.*$"
 # Your stuff...
 # ------------------------------------------------------------------------------
+
 # Are you using local passwords or oidc?
 AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower()
 

docs/environment_variables.md

@@ -1,76 +1,60 @@
-## Environment Variables to Set for Deployment
-[comment]: # Which branch of SCRAM to use (you probably want to set it to a release tag)
-scram_code_branch:
-#### Systems
-[comment]: # Email of the main admin
-scram_manager_email:
-[comment]: # Set to true for production mode; set to false to set up the compose.override.local.yml stack
-scram_prod: true
-[comment]: # Set to true if you want ansible to install a scram user
-scram_install_user: true
-[comment]: # What group to put `scram` user in
-scram_group: 'scram'
-[comment]: # What username to use for `scram` user
-scram_user: ''
-[comment]: # WHat uid to use for `scram` user
-scram_uid: ''
-[comment]: # What directory to use for base of the repo
-scram_home: '/usr/local/scram'
-[comment]: # IP or DNS record for your postgres host
-scram_postgres_host:
-[comment]: # What postgres user to use
-scram_postgres_user: ''
+# Environment Variables Reference
 
-#### Authentication
-[comment]: # This chooses if you want to use oidc or local accounts. This can be local or oidc only. Default: `local`
-scram_auth_method: "local"
-[comment]: # This client id (username) for your oidc connection. Only need to set this if you are trying to do oidc.
-scram_oidc_client_id: 
+To update, run `make update-env-docs`.
 
-#### Networking
-[comment]: # What is the peering interface docker uses for gobgp to talk to the router
-scram_peering_iface: 'ens192'
-[comment]: # The v6 network of your peering connection
-scram_v4_subnet: '10.0.0.0/24'
-[comment]: # The v4 IP of the peering connection for the router side
-scram_v4_gateway: '10.0.0.1'
-[comment]: # The v4 IP of the peering connection for gobgp side
-scram_v4_address: '10.0.0.2'
-[comment]: # The v6 network of your peering connection
-scram_v6_subnet: '2001:db8::/64'
-[comment]: # The v6 IP of the peering connection for the router side
-scram_v6_gateway: '2001:db8::2'
-[comment]: # The v6 IP of the peering connection for the gobgp side
-scram_v6_address: '2001:db8::3'
-[comment]: # The AS you want to use for gobgp
-scram_as: 
-[comment]: # A string representing your gobgp instance. Often seen as the local IP of the gobgp instance
-scram_router_id:
-[comment]: #
-scram_peer_as:
-[comment]: # The AS you want to use for gobgp side (can this be the same as `scram_as`?)
-scram_local_as:
-[comment]: # The fqdn of the server hosting this - to be used for nginx
-scram_nginx_host:
-[comment]: # List of allowed hosts per the django setting "ALLOWED_HOSTS". This should be a list of strings in shell
-[comment]: # `django` is required for the websockets to work
-[comment]: # Our Ansible assumes `django` + `scram_nginx_host`
-scram_django_allowed_hosts: "django"
-[comment]: # The fqdn of the server hosting this - to be used for nginx
-scram_server_alias:
-[comment]: # Do you want to set an md5 for authentication of bgp
-scram_bgp_md5_enabled: false
-[comment]: # The neighbor config of your gobgp config
-scram_neighbors:
-[comment]: # The v6 address of your neighbor
-  - neighbor_address: 2001:db8::2
-[comment]: # This is a v6 address so don't use v4
-    ipv4: false
-[comment]: # This is a v6 address so use v6
-    ipv6: true
-[comment]: # The v4 address of your neighbor
-  - neighbor_address: 10.0.0.200
-[comment]: # This is a v4 address so use v4
-    ipv4: true
-[comment]: # This is a v4 address so don't use v6
-    ipv6: false
+| Variable | Service | Environments | Default | file | Description |
+| --- | --- | --- | --- | --- | --- |
+| `CELERY_BEAT_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - |
+| `CELERY_WORKER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - |
+| `DEBUG` | Compose | Multiple | - | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | Here we setup a debugger if this is desired. This obviously should not be run in production |
+| `DJANGO_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - |
+| `DOCS_PORT` | Compose | Multiple | 8888 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | - |
+| `FLOWER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - |
+| `GOBGP_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - |
+| `HOSTNAME` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `POSTGRES_ENABLED` | Compose | Common | 1 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.production.yml](file://compose.override.production.yml), [compose.override.yml](file://compose.override.yml), [compose.yml](file://compose.yml) | - |
+| `REDIS_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - |
+| `SCRAM_PEERING_IFACE` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V4_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V4_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V4_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V6_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V6_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `SCRAM_V6_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - |
+| `TRANSLATOR_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - |
+| `CONN_MAX_AGE` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | noqa F405 |
+| `DATABASE_URL` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/production.py](file://config/settings/production.py) | DATABASES https docs.djangoproject.com/en/dev/ref/settings databases |
+| `DEBUG` | Django | Unknown | - | [config/asgi.py](file://config/asgi.py) | Here we setup a debugger if this is desired. This obviously should not be run in production |
+| `DJANGO_ADMIN_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | ADMIN Django Admin URL regex |
+| `DJANGO_ALLOWED_HOSTS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings allowed-hosts |
+| `DJANGO_DEFAULT_FROM_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings default-from-email |
+| `DJANGO_EMAIL_BACKEND` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/local.py](file://config/settings/local.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings email-backend |
+| `DJANGO_EMAIL_SUBJECT_PREFIX` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings email-subject-prefix |
+| `DJANGO_READ_DOT_ENV_FILE` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - |
+| `DJANGO_SECURE_CONTENT_TYPE_NOSNIFF` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/middleware x-content-type-options-nosniff |
+| `DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-include-subdomains |
+| `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload |
+| `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect |
+| `DJANGO_SERVER_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email |
+| `DJANGO_SETTINGS_MODULE` | Django | Unknown | - | [config/wsgi.py](file://config/wsgi.py) | os.environ DJANGO_SETTINGS_MODULE = "config.settings.production" # noqa ERA001 |
+| `OIDC_RP_CLIENT_ID` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - |
+| `OIDC_RP_CLIENT_SECRET` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - |
+| `POSTGRES_SSL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - |
+| `REDIS_HOST` | Django | Common | "redis" | [config/settings/base.py](file://config/settings/base.py) | - |
+| `REDIS_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - |
+| `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? |
+| `USE_DOCKER` | Django | Local | - | [config/settings/local.py](file://config/settings/local.py) | - |
+| `BAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR |
+| `DEFAULT_VAR` | Other | Test | 'my_default' | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | Has default |
+| `DJANGO_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - |
+| `ENV_VAR` | Other | Test | "env_def" | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - |
+| `FOO` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR |
+| `STANDARD_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | This is standard |
+| `STRICT_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - |
+| `CELERY_BROKER_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - |
+| `CELERY_RESULT_BACKEND` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - |
+| `DISABLE_PROCESS_UPDATES` | Scheduler | Test | - | [scheduler/tests/test_app.py](file://scheduler/tests/test_app.py) | Set the disable env var and then reload settings, then the app |
+| `SCRAM_API_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - |
+| `DEBUG` | Translator | Unknown | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production |
+| `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - |
+| `SCRAM_HOSTNAME` | Translator | Unknown | "scram_hostname_not_set" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash |

pyproject.toml

@@ -1,6 +1,7 @@
 [project]
 name = "SCRAM"
 version = "1.5.1"
+requires-python = ">=3.12"
 
 # ==== pytest ====
 [tool.pytest.ini_options]