Merge upstream 26 01 30

.github/workflows/release.yaml

@@ -63,7 +63,7 @@ jobs:
         token: ${{ steps.generate-token.outputs.token }}
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.3
+      uses: prefix-dev/setup-pixi@v0.9.4
       with:
         pixi-version: v0.59.0
         cache: true

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Test workflows
+name: Test
 
 on:
   push:
@@ -17,8 +17,30 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-tests:
-    name: OS
+  unit-tests:
+    name: Unit
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Setup Pixi
+      uses: prefix-dev/setup-pixi@v0.9.4
+      with:
+        pixi-version: v0.59.0
+        cache: true
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
+
+    - name: Run unit tests
+      run: |
+        pixi run unit-tests
+
+  integration-tests:
+    name: Integration
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
@@ -43,7 +65,6 @@ jobs:
             - 'data/**'
             - 'Snakefile'
             - 'config/**'
-            - 'test/**'
             - 'pixi.toml'
             - 'pixi.lock'
             - '.github/workflows/test.yaml'
@@ -58,31 +79,41 @@ jobs:
         sudo docker builder prune -a --force
         echo "Final disk space"
         df -h
+
     - name: Skip - no source changes
       if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
       run: echo "Skipping tests because no source code changes detected"
 
     - name: Setup Pixi
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
-      uses: prefix-dev/setup-pixi@v0.9.3
+      uses: prefix-dev/setup-pixi@v0.9.4
       with:
         pixi-version: v0.59.0
         cache: true
-        # Do not cache in branches
-        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
 
     - name: Setup cache keys
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
-        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data
+        echo "MONTH=$(date +'%Y%m')" >> $GITHUB_ENV # cutouts
+        echo "VERSIONS_HASH=${{ hashFiles('data/versions.csv') }}" >> $GITHUB_ENV
 
     - uses: actions/cache@v5
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       with:
-        path: |
-          data
-          cutouts
-        key: data-cutouts-${{ env.WEEK }}
+        path: data
+        key: data-${{ env.WEEK }}-${{ env.VERSIONS_HASH }}
+
+    - uses: actions/cache@v5
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      with:
+        path: cutouts
+        key: cutouts-${{ env.MONTH }}
+
+    - name: Restore git-tracked files in data/
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      run: git checkout HEAD -- data/
 
     - name: Run pylint check on scripts
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
@@ -98,11 +129,6 @@ jobs:
       run: |
         pixi run integration-tests
 
-    - name: Run unit tests
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
-      run: |
-        pixi run unit-tests
-
     - name: Upload artifacts
       if: always()
       uses: actions/upload-artifact@v6

.github/workflows/update-lockfile.yaml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v6
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.3
+      uses: prefix-dev/setup-pixi@v0.9.4
       with:
         pixi-version: v0.59.0
 

.gitignore

@@ -31,6 +31,7 @@ gurobi.log
 /cutouts
 /tmp
 doc/_build
+_build
 
 /scripts/old
 /scripts/create_scenarios.py
@@ -65,6 +66,8 @@ d1gam3xoknrgr2.cloudfront.net/
 
 *~
 
+*.egg-info
+
 *.pyc
 
 *.xlsx

.pre-commit-config.yaml

@@ -1,8 +1,7 @@
 # SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: CC0-1.0
-
-exclude: "^LICENSES"
+exclude: "^LICENSES|^config/schema\\.json$|^config/config\\.default\\.yaml$"
 
 ci:
   autoupdate_schedule: quarterly

Snakefile

@@ -7,8 +7,11 @@ import yaml
 import sys
 from os.path import normpath, exists, join
 from shutil import copyfile, move, rmtree, unpack_archive
+from dotenv import load_dotenv
 from snakemake.utils import min_version
 
+load_dotenv()
+
 min_version("8.11")
 
 from scripts._helpers import (
@@ -17,13 +20,16 @@ from scripts._helpers import (
     get_shadow,
     path_provider,
 )
+from scripts.lib.validation.config import validate_config
 
 
 configfile: "config/config.default.yaml"
 configfile: "config/plotting.default.yaml"
 configfile: "config/config.de.yaml"
 
 
+validate_config(config)
+
 run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)

config/config.de.yaml

@@ -417,6 +417,7 @@ solving:
   runtime: 12h
   mem_mb: 70000 #30000 is OK for 22 nodes, 365H; 140000 for 22 nodes 3H; 400000 for 44 nodes 3H
   options:
+    custom_extra_functionality: "../scripts/pypsa-de/additional_functionality.py"
     assign_all_duals: true
     load_shedding: false
     skip_iterations: true # settings for post-discretization: false