diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 07d330243d..b3abdcd02d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,7 +54,7 @@ jobs:
 
     runs-on: ubuntu-24.04
     timeout-minutes: 25
-    needs: [lint, security]
+    needs: [lint]
 
     steps:
     - uses: actions/checkout@v4
@@ -122,7 +122,7 @@ jobs:
 
   native_tests:
 
-    needs: [lint, security]
+    needs: [lint]
     permissions:
       contents: read
       id-token: write
@@ -141,9 +141,6 @@ jobs:
             ]
           }' || '{
             "include": [
-              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"},
-              {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"},
-              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-llfuse"},
               {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-llfuse", "binary": "borg-linux-glibc235-x86_64-gh"},
               {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-llfuse", "binary": "borg-linux-glibc235-arm64-gh"},
               {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"},
@@ -182,6 +179,15 @@ jobs:
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
+    - name: Cache tox environments
+      uses: actions/cache@v4
+      with:
+        path: .tox
+        key: ${{ runner.os }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.txt', 'pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-tox-${{ matrix.toxenv }}-
+          ${{ runner.os }}-tox-
+
     - name: Install Linux packages
       if: ${{ runner.os == 'Linux' }}
       run: |
@@ -203,7 +209,7 @@ jobs:
         brew bundle install
 
     - name: Configure OpenSSH SFTP server (test only)
-      if: ${{ runner.os == 'Linux' }}
+      if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
       run: |
         sudo mkdir -p /run/sshd
         sudo useradd -m -s /bin/bash sftpuser || true
@@ -233,7 +239,7 @@ jobs:
         echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV
 
     - name: Install and configure MinIO S3 server (test only)
-      if: ${{ runner.os == 'Linux' }}
+      if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
       run: |
         set -e
         arch=$(uname -m)
@@ -349,7 +355,7 @@ jobs:
       attestations: write
     runs-on: ubuntu-24.04
     timeout-minutes: 90
-    needs: [lint, security]
+    needs: [lint]
     continue-on-error: true
 
     strategy:
@@ -547,7 +553,7 @@ jobs:
     if: true  # can be used to temporarily disable the build
     runs-on: windows-latest
     timeout-minutes: 90
-    needs: [lint, security]
+    needs: [lint]
 
     env:
       PY_COLORS: 1