diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1f17e37..fa5f2115 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,15 @@ repos: - id: ruff-check args: [--fix, --exit-non-zero-on-fix] + - repo: local + hooks: + - id: update-env-docs + name: Check Environment Variables Markdown + entry: make update-env-docs CHECK=true + language: system + pass_filenames: false + always_run: true + - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.10.7 hooks: diff --git a/Makefile b/Makefile index 2f46ef26..7f887458 100644 --- a/Makefile +++ b/Makefile @@ -175,4 +175,12 @@ copy-libs: @docker compose cp translator:/app/attribute_pb2_grpc.py translator/ @docker compose cp translator:/app/capability_pb2.py translator/ @docker compose cp translator:/app/capability_pb2.pyi translator/ - @docker compose cp translator:/app/capability_pb2_grpc.py translator/ + +## update-env-docs: update environment variable documentation append CHECK=true to get a diff if not up to date +.Phony: update-env-docs +update-env-docs: +ifeq ($(CHECK),true) + @uv run scripts/extract_env_vars.py --check +else + @uv run scripts/extract_env_vars.py +endif diff --git a/compose.yml b/compose.yml index 8c6b6cb6..a515fb3b 100644 --- a/compose.yml +++ b/compose.yml @@ -1,5 +1,4 @@ --- - services: django: build: diff --git a/config/settings/base.py b/config/settings/base.py index 289f5f59..8bfc90e8 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -290,6 +290,7 @@ CORS_URLS_REGEX = r"^/api/.*$" # Your stuff... # ------------------------------------------------------------------------------ + # Are you using local passwords or oidc? AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() diff --git a/docs/environment_variables.md b/docs/environment_variables.md index c44629a7..6d36dbd8 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -1,76 +1,60 @@ -## Environment Variables to Set for Deployment -[comment]: # Which branch of SCRAM to use (you probably want to set it to a release tag) -scram_code_branch: -#### Systems -[comment]: # Email of the main admin -scram_manager_email: -[comment]: # Set to true for production mode; set to false to set up the compose.override.local.yml stack -scram_prod: true -[comment]: # Set to true if you want ansible to install a scram user -scram_install_user: true -[comment]: # What group to put `scram` user in -scram_group: 'scram' -[comment]: # What username to use for `scram` user -scram_user: '' -[comment]: # WHat uid to use for `scram` user -scram_uid: '' -[comment]: # What directory to use for base of the repo -scram_home: '/usr/local/scram' -[comment]: # IP or DNS record for your postgres host -scram_postgres_host: -[comment]: # What postgres user to use -scram_postgres_user: '' +# Environment Variables Reference -#### Authentication -[comment]: # This chooses if you want to use oidc or local accounts. This can be local or oidc only. Default: `local` -scram_auth_method: "local" -[comment]: # This client id (username) for your oidc connection. Only need to set this if you are trying to do oidc. -scram_oidc_client_id: +To update, run `make update-env-docs`. -#### Networking -[comment]: # What is the peering interface docker uses for gobgp to talk to the router -scram_peering_iface: 'ens192' -[comment]: # The v6 network of your peering connection -scram_v4_subnet: '10.0.0.0/24' -[comment]: # The v4 IP of the peering connection for the router side -scram_v4_gateway: '10.0.0.1' -[comment]: # The v4 IP of the peering connection for gobgp side -scram_v4_address: '10.0.0.2' -[comment]: # The v6 network of your peering connection -scram_v6_subnet: '2001:db8::/64' -[comment]: # The v6 IP of the peering connection for the router side -scram_v6_gateway: '2001:db8::2' -[comment]: # The v6 IP of the peering connection for the gobgp side -scram_v6_address: '2001:db8::3' -[comment]: # The AS you want to use for gobgp -scram_as: -[comment]: # A string representing your gobgp instance. Often seen as the local IP of the gobgp instance -scram_router_id: -[comment]: # -scram_peer_as: -[comment]: # The AS you want to use for gobgp side (can this be the same as `scram_as`?) -scram_local_as: -[comment]: # The fqdn of the server hosting this - to be used for nginx -scram_nginx_host: -[comment]: # List of allowed hosts per the django setting "ALLOWED_HOSTS". This should be a list of strings in shell -[comment]: # `django` is required for the websockets to work -[comment]: # Our Ansible assumes `django` + `scram_nginx_host` -scram_django_allowed_hosts: "django" -[comment]: # The fqdn of the server hosting this - to be used for nginx -scram_server_alias: -[comment]: # Do you want to set an md5 for authentication of bgp -scram_bgp_md5_enabled: false -[comment]: # The neighbor config of your gobgp config -scram_neighbors: -[comment]: # The v6 address of your neighbor - - neighbor_address: 2001:db8::2 -[comment]: # This is a v6 address so don't use v4 - ipv4: false -[comment]: # This is a v6 address so use v6 - ipv6: true -[comment]: # The v4 address of your neighbor - - neighbor_address: 10.0.0.200 -[comment]: # This is a v4 address so use v4 - ipv4: true -[comment]: # This is a v4 address so don't use v6 - ipv6: false +| Variable | Service | Environments | Default | file | Description | +| --- | --- | --- | --- | --- | --- | +| `CELERY_BEAT_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `CELERY_WORKER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `DEBUG` | Compose | Multiple | - | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `DJANGO_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `DOCS_PORT` | Compose | Multiple | 8888 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | - | +| `FLOWER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `GOBGP_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `HOSTNAME` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `POSTGRES_ENABLED` | Compose | Common | 1 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.production.yml](file://compose.override.production.yml), [compose.override.yml](file://compose.override.yml), [compose.yml](file://compose.yml) | - | +| `REDIS_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `SCRAM_PEERING_IFACE` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `TRANSLATOR_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `CONN_MAX_AGE` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | noqa F405 | +| `DATABASE_URL` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/production.py](file://config/settings/production.py) | DATABASES https docs.djangoproject.com/en/dev/ref/settings databases | +| `DEBUG` | Django | Unknown | - | [config/asgi.py](file://config/asgi.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `DJANGO_ADMIN_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | ADMIN Django Admin URL regex | +| `DJANGO_ALLOWED_HOSTS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings allowed-hosts | +| `DJANGO_DEFAULT_FROM_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings default-from-email | +| `DJANGO_EMAIL_BACKEND` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/local.py](file://config/settings/local.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings email-backend | +| `DJANGO_EMAIL_SUBJECT_PREFIX` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings email-subject-prefix | +| `DJANGO_READ_DOT_ENV_FILE` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `DJANGO_SECURE_CONTENT_TYPE_NOSNIFF` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/middleware x-content-type-options-nosniff | +| `DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-include-subdomains | +| `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload | +| `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect | +| `DJANGO_SERVER_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email | +| `DJANGO_SETTINGS_MODULE` | Django | Unknown | - | [config/wsgi.py](file://config/wsgi.py) | os.environ DJANGO_SETTINGS_MODULE = "config.settings.production" # noqa ERA001 | +| `OIDC_RP_CLIENT_ID` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `OIDC_RP_CLIENT_SECRET` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `POSTGRES_SSL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | +| `REDIS_HOST` | Django | Common | "redis" | [config/settings/base.py](file://config/settings/base.py) | - | +| `REDIS_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | +| `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? | +| `USE_DOCKER` | Django | Local | - | [config/settings/local.py](file://config/settings/local.py) | - | +| `BAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | +| `DEFAULT_VAR` | Other | Test | 'my_default' | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | Has default | +| `DJANGO_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `ENV_VAR` | Other | Test | "env_def" | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `FOO` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | +| `STANDARD_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | This is standard | +| `STRICT_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `CELERY_BROKER_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `CELERY_RESULT_BACKEND` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `DISABLE_PROCESS_UPDATES` | Scheduler | Test | - | [scheduler/tests/test_app.py](file://scheduler/tests/test_app.py) | Set the disable env var and then reload settings, then the app | +| `SCRAM_API_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `DEBUG` | Translator | Unknown | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - | +| `SCRAM_HOSTNAME` | Translator | Unknown | "scram_hostname_not_set" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash | diff --git a/pyproject.toml b/pyproject.toml index 2cc81574..32f24455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "SCRAM" version = "1.5.1" +requires-python = ">=3.12" # ==== pytest ==== [tool.pytest.ini_options] diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py new file mode 100755 index 00000000..196634de --- /dev/null +++ b/scripts/extract_env_vars.py @@ -0,0 +1,341 @@ +"""Script to extract environment variables.""" + +# !/usr/bin/env uv run +import argparse +import difflib +import logging +import re +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Exclusion patterns for directories +EXCLUDE_DIRS = { + "venv", + ".venv", + ".git", + ".idea", + ".vscode", + ".pytest_cache", + ".ruff_cache", + "staticfiles", + "node_modules", + ".envs", + "__pycache__", +} + +# var_name, default_value from python files +PYTHON_ENV_PATTERNS = [ + ( + r'env(?:\.\w+)?\(\s*["\']([^"\']+)["\'](?:,\s*default=(?:get_random_secret_key\(\)|' + r'get_random_string\(50, allowed_chars="abcdefghijklmnopqrstuvwxyz0123456789"\)|' + r"[^,\)]+))?\s*\)" + ), + r'os\.getenv\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', + r'os\.environ\.get\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', + r'os\.environ\[\s*["\']([^"\']+)["\']\s*\]', +] + +# var_name, default_value from compose files +COMPOSE_ENV_PATTERN = r"\$\{([^}:-]+)(?::-([^}]*))?\}" + + +def extract_comment(lines: list[str], line_index: int) -> str: + """Pull comments from either the same line or right above to add context. + + Returns: + str: comment relevant to the envvar line + """ + # Check same line comments + current_line = lines[line_index] + if "#" in current_line: + comment = current_line.split("#", 1)[1].strip() + if comment: + return comment + + # Check lines above + comments_above = [] + # Start at the line above and continue up the lines until you find a line without a comment + # or you hit the beginning of the file + for i in range(line_index - 1, -1, -1): + prev_line = lines[i].strip() + if prev_line.startswith("#"): + comments_above.insert(0, prev_line.lstrip("#").strip()) + else: + break + + if comments_above: + return " ".join(comments_above) + return "" + + +def clean_comment(comment: str) -> str: + """Remove heavily repeated special characters which are mostly useless in docs. + + Returns: + str: a comment without special characters + """ + if not comment: + return "" + comment = re.sub(r"[^a-zA-Z0-9\s]{2,}", " ", comment) + comment = comment.strip(" #=-_*.") + comment = re.sub(r"\s+", " ", comment) + return comment.strip() + + +def extract_from_python(content: str, file_path: Path) -> dict[str, dict[str, Any]]: + """Extract environment variables from Python files. + + Returns: + dict: python environment variables + """ + vars_found: dict[str, dict[str, Any]] = {} + lines = content.splitlines() + for i, line in enumerate(lines): + for pattern in PYTHON_ENV_PATTERNS: + # finditer so that we have loop over the matches as dicts + # (match.group(1) is the var name, match.group(2) is the default value) + matches = re.finditer(pattern, line) + for match in matches: + var_name = match.group(1) + default_value = match.group(2) if len(match.groups()) > 1 else None + comment = clean_comment(extract_comment(lines, i)) + + if var_name not in vars_found: + vars_found[var_name] = {"default": default_value, "desc": comment, "file": {str(file_path)}} + # update comment if there wasnt one in previous findings of the var + elif comment and not vars_found[var_name]["desc"]: + vars_found[var_name]["desc"] = comment + return vars_found + + +def extract_from_compose(content: str, file_path: Path) -> dict[str, dict[str, Any]]: + """Extract environment variables from Compose files. + + Returns: + dict: compose environment variables + """ + vars_found: dict[str, dict[str, Any]] = {} + lines = content.splitlines() + for i, line in enumerate(lines): + matches = re.finditer(COMPOSE_ENV_PATTERN, line) + for match in matches: + var_name = match.group(1) + default_value = match.group(2) if len(match.groups()) > 1 else None + comment = clean_comment(extract_comment(lines, i)) + + if var_name not in vars_found: + vars_found[var_name] = {"default": default_value, "desc": comment, "file": {str(file_path)}} + elif comment and not vars_found[var_name]["desc"]: + vars_found[var_name]["desc"] = comment + return vars_found + + +def infer_environment(file_path: Path) -> str: + """Infer the environment from the file path. + + Returns: + str: the type of environment + """ + path_str = str(file_path) + if "production" in path_str: + return "Production" + if "local" in path_str: + return "Local" + if "test" in path_str: + return "Test" + if "settings/base" in path_str or "shared" in path_str or "common" in path_str or path_str == "compose.yml": + return "Common" + return "Unknown" + + +def get_service(file_path: Path) -> str: + """Determine the service name based on the file path. + + Returns: + str: the name of the service/app + """ + path_parts = Path(file_path).parts + if "config" in path_parts: + return "Django" + if "translator" in path_parts: + return "Translator" + if "scheduler" in path_parts: + return "Scheduler" + if "compose" in path_parts or Path(file_path).name.startswith("compose"): + return "Compose" + return "Other" + + +def parse_existing_docs(output_path: Path) -> dict[str, str]: + """Parse existing documentation to preserve manual descriptions. + + Returns: + dict: a dictionary with existing descriptions + """ + manual_descs: dict[str, str] = {} + if not output_path.exists(): + return manual_descs + + content = output_path.read_text(encoding="utf-8") + rows = re.findall(r"\| `([^`]+)` \| [^|]+ \| [^|]+ \| [^|]+ \| [^|]+ \| ([^|]*)\|", content) + for var, desc in rows: + clean_desc = clean_comment(desc.strip()) + if clean_desc and clean_desc not in {"-", "Manually added description"}: + manual_descs[var] = clean_desc + return manual_descs + + +def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: # noqa: C901 + """Scan the project directory for environment variables. + + Returns: + dict: a dictionary with all the environment variables + """ + all_vars: dict[tuple[str, str], dict[str, Any]] = {} + + for path in root_dir.rglob("*"): + if any(exclude in path.parts for exclude in EXCLUDE_DIRS): + continue + + rel_path = path.relative_to(root_dir) + + if path.suffix == ".py": + try: + content = path.read_text() + vars_found = extract_from_python(content, rel_path) + for var, info in vars_found.items(): + service = get_service(rel_path) + key = (var, service) + if key not in all_vars: + all_vars[key] = { + "envs": set(), + "default": info["default"], + "desc": info["desc"], + "file": set(), + } + all_vars[key]["envs"].add(infer_environment(rel_path)) + all_vars[key]["file"].update(info["file"]) + if info["desc"] and not all_vars[key]["desc"]: + all_vars[key]["desc"] = info["desc"] + except Exception: + logger.exception("Error reading %s", path) + + elif path.suffix in {".yml", ".yaml"} and "compose" in path.name: + try: + content = path.read_text() + vars_found = extract_from_compose(content, rel_path) + for var, info in vars_found.items(): + key = (var, "Compose") + if key not in all_vars: + all_vars[key] = { + "envs": set(), + "default": info["default"], + "desc": info["desc"], + "file": set(), + } + all_vars[key]["envs"].add(infer_environment(rel_path)) + all_vars[key]["file"].update(info["file"]) + if info["desc"] and not all_vars[key]["desc"]: + all_vars[key]["desc"] = info["desc"] + except Exception: + logger.exception("Error reading %s", path) + + return all_vars + + +def sort_by_service_and_name(item: tuple[tuple[str, str], dict[str, Any]]) -> tuple[str, str]: + """Sort by Service Name first, then Variable Name. + + Returns: + tuple: of the service name and variable name + """ + (var_name, service), _metadata = item + return service, var_name + + +def generate_markdown_content(all_vars: dict[tuple[str, str], dict[str, Any]], manual_descs: dict[str, str]) -> str: + """Generate the markdown content for the environment variables documentation. + + Returns: + str: the markdown content + """ + sorted_vars = sorted(all_vars.items(), key=sort_by_service_and_name) + + lines = [ + "# Environment Variables Reference", + "", + "To update, run `make update-env-docs`.", + "", + "| Variable | Service | Environments | Default | file | Description |", + "| --- | --- | --- | --- | --- | --- |", + ] + + for (var_name, service), data in sorted_vars: + if "Common" in data["envs"]: + envs = "Common" + elif len(data["envs"]) > 1: + envs = "Multiple" + else: + envs = ", ".join(sorted(data["envs"])) + + # Grab the default value or fall back to "-" + default = data["default"] if data["default"] else "-" + # Used to create a link to the file + file = ", ".join(f"[{f}](file://{f})" for f in sorted(data["file"])) + + # Use manual description if available, otherwise use code comments + description = manual_descs.get(var_name, data["desc"]) + if not description: + description = "-" + + lines.append(f"| `{var_name}` | {service} | {envs} | {default} | {file} | {description} |") + + return "\n".join(lines) + "\n" + + +def main() -> None: + """Main function to parse arguments and orchestrate the extraction.""" + parser = argparse.ArgumentParser(description="Extract environment variables from SCRAM project.") + parser.add_argument("--check", action="store_true", help="Check if documentation is up to date.") + args = parser.parse_args() + + root_dir = Path(__file__).resolve().parent.parent + output_path = root_dir / "docs/environment_variables.md" + + manual_descs = parse_existing_docs(output_path) + all_vars = find_env_vars(root_dir) + new_content = generate_markdown_content(all_vars, manual_descs) + + if args.check: + if output_path.exists(): + current_content = output_path.read_text() + if current_content == new_content: + logger.info("Documentation is up to date.") + return + logger.info("Documentation is out of date:\n") + diff = difflib.unified_diff( + current_content.splitlines(), + new_content.splitlines(), + fromfile="docs/environment_variables.md (Current)", + tofile="docs/environment_variables.md (Generated)", + lineterm="", + ) + logger.warning("\n".join(diff)) + sys.exit(1) + else: + logger.warning("Documentation file docs/environment_variables.md does not exist!") + sys.exit(1) + elif output_path.exists() and output_path.read_text() == new_content: + logger.info("Documentation is already up to date. No changes made.") + else: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(new_content) + logger.info("Updated docs/environment_variables.md") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_extract_env_vars.py b/scripts/tests/test_extract_env_vars.py new file mode 100644 index 00000000..5bf45745 --- /dev/null +++ b/scripts/tests/test_extract_env_vars.py @@ -0,0 +1,104 @@ +"""Define tests for our extract env vars script.""" + +from pathlib import Path + +from scripts.extract_env_vars import ( + clean_comment, + extract_comment, + extract_from_compose, + extract_from_python, + get_service, + infer_environment, +) + + +def test_extract_comment() -> None: + """Test extracting comments from adjacent or identical lines.""" + lines = [" # A useful comment", " VAR = os.getenv('FOO') # Same line comment", " VAR2 = os.getenv('BAR')"] + + assert extract_comment(lines, 1) == "Same line comment" + assert not extract_comment(lines, 2) + assert extract_comment(lines, 0) == "A useful comment" + + +def test_clean_comment() -> None: + """Test cleaning up special characters and excess whitespace from comments.""" + assert clean_comment(" # -- My comment ** ") == "My comment" + assert not clean_comment("") + assert not clean_comment(None) + + +def test_extract_from_python() -> None: + """Test extracting environment variables from Python source code strings.""" + content = """ + # This is standard + os.getenv("STANDARD_VAR") + + os.getenv('DEFAULT_VAR', 'my_default') # Has default + + os.environ.get("ENV_VAR", "env_def") + + os.environ["STRICT_VAR"] + + env.str("DJANGO_VAR", default="django_def") + """ + + dummy_path = Path("dummy.py") + result = extract_from_python(content, dummy_path) + + assert "STANDARD_VAR" in result + assert result["STANDARD_VAR"]["default"] is None + assert result["STANDARD_VAR"]["desc"] == "This is standard" + + assert "DEFAULT_VAR" in result + assert result["DEFAULT_VAR"]["default"] == "'my_default'" + assert result["DEFAULT_VAR"]["desc"] == "Has default" + + assert "ENV_VAR" in result + assert result["ENV_VAR"]["default"] == '"env_def"' + + assert "STRICT_VAR" in result + + assert "DJANGO_VAR" in result + assert result["DJANGO_VAR"]["default"] is None + + +def test_extract_from_compose() -> None: + """Test extracting environment variables and defaults from Compose YAML strings.""" + content = """ + services: + app: + environment: + - SIMPLE_VAR=${SIMPLE_VAR} # Simple description + - DEFAULT_VAR=${DEFAULT_VAR:-fallback_value} + """ + + dummy_path = Path("compose.yml") + result = extract_from_compose(content, dummy_path) + + assert "SIMPLE_VAR" in result + assert result["SIMPLE_VAR"]["default"] is None + assert result["SIMPLE_VAR"]["desc"] == "Simple description" + + assert "DEFAULT_VAR" in result + assert result["DEFAULT_VAR"]["default"] == "fallback_value" + + +def test_infer_environment() -> None: + """Test inferring the environment label based on specific file paths.""" + assert infer_environment(Path("config/settings/production.py")) == "Production" + assert infer_environment(Path("config/settings/local.py")) == "Local" + assert infer_environment(Path("tests/test_something.py")) == "Test" + assert infer_environment(Path("config/settings/base.py")) == "Common" + assert infer_environment(Path("translator/shared.py")) == "Common" + assert infer_environment(Path("compose.yml")) == "Common" + assert infer_environment(Path("random_file.py")) == "Unknown" + + +def test_get_service() -> None: + """Test identifying the service name based on directory structure or file name.""" + assert get_service(Path("config/settings.py")) == "Django" + assert get_service(Path("translator/app.py")) == "Translator" + assert get_service(Path("scheduler/tasks.py")) == "Scheduler" + assert get_service(Path("compose.override.yml")) == "Compose" + assert get_service(Path("scripts/extract_env_vars.py")) == "Other"