Switch to pixi (#802)

* Create draft of pixi.toml * Modify contributing docs to reflect new install process * Auto-update lockfile * Make CI unit tests work with pixi * Add lockfile * Add psutil to test deps * Add oldies tests * Fix test environments * Mske pre-commit hooks use pixi environments * Only use -nauto in CI * Add doctest * Remove update section from pixi.toml * Change ci job name * Remove multiple postinstalls * Pixify daily tests * Find minimum versions that actually work * Make lint environment leaner * Add formlulaic minimum version to oldies * Only run pre-commit on changes in CI * Pin pixi-setup to sha * Fix deps * Minor fixes * Use intel mkl in benchmarks * Try to make docs work * Make installing nigthlies a pixi task (for local use, too) * Fix pyarrow nighly installation * Add test feature to nightly environment * Add environment for simultaneous editable tabmat and glum install * Update dev dependencies * fix mypy quirk * fix mypy typing errors * Address @pavelzw's comments * Remove reference to conda/mamba from contributing instructions Co-authored-by: Jan Tilly <jan.tilly@quantco.com> --------- Co-authored-by: Marc-Antoine Schmidt <marc-antoine.schmidt@quantco.com> Co-authored-by: Jan Tilly <jan.tilly@quantco.com>
Quantco · Jun 25, 2024 · 2684b86 · 2684b86
1 parent 817ac80
commit 2684b86
Show file tree

Hide file tree

Showing 20 changed files with 25,019 additions and 218 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,75 +3,54 @@ on: [push]
 
 jobs:
   pre-commit-checks:
-    name: Linux - pre-commit checks - Python 3.11
+    name: Linux - pre-commit checks
     timeout-minutes: 30
     runs-on: ubuntu-latest
     env:
       PRE_COMMIT_USE_MICROMAMBA: 1
     steps:
       - name: Checkout branch
         uses: actions/checkout@v4
-      - name: Set up micromamba
-        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822
-      - name: Add micromamba to GITHUB_PATH
-        run: echo "${HOME}/micromamba-bin" >> "$GITHUB_PATH"
-      - name: Install Python 3.11
-        uses: actions/setup-python@v5
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@v0.8.0
         with:
-          python-version: "3.11"
-      - name: Run pre-commit checks
-        uses: pre-commit/action@v3.0.1
+          environments: lint default
+      - name: pre-commit
+        run: pixi run -e lint pre-commit-run --color=always --show-diff-on-failure
 
   unit-tests:
-    name: Tests - ${{ matrix.os }} - Py${{ matrix.python-version }}
+    name: Tests
     runs-on: ${{ matrix.os }}
     env:
       CI: True
     strategy:
       fail-fast: true
       matrix:
         include:
-          - { os: ubuntu-latest, python-version: '3.9' }
-          - { os: ubuntu-latest, python-version: '3.10' }
-          - { os: ubuntu-latest, python-version: '3.11' }
-          - { os: ubuntu-latest, python-version: '3.12' }
-          - { os: macos-latest, python-version: '3.12' }
-          - { os: windows-latest, python-version: '3.12' }
+          - { os: ubuntu-latest, environment: 'py39' }
+          - { os: ubuntu-latest, environment: 'py310' }
+          - { os: ubuntu-latest, environment: 'py311' }
+          - { os: ubuntu-latest, environment: 'py312' }
+          - { os: ubuntu-latest, environment: 'oldies' }
+          - { os: windows-latest, environment: 'py312' }
+          - { os: macos-latest, environment: 'py312' }
     steps:
       - name: Checkout branch
         uses: actions/checkout@v4
-      - name: Set up conda env
-        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
         with:
-          environment-file: environment.yml
-          init-shell: ${{ matrix.os == 'windows-latest' && 'powershell' || 'bash' }}
-          cache-environment: true
-          create-args: >-
-            python=${{ matrix.python-version }}
-      - name: Install repository (unix)
-        if: matrix.os != 'windows-latest'
-        shell: bash -el {0}
-        run: pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
-      - name: Install repository (windows)
-        if: matrix.os == 'windows-latest'
-        shell: powershell
-        run: pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
-      - name: Run pytest (unix)
-        if: matrix.os != 'windows-latest'
-        shell: bash -el {0}
-        run: pytest -nauto tests/glm --doctest-modules src/glum
-      - name: Run pytest (windows)
-        if: matrix.os == 'windows-latest'
-        shell: powershell
-        run: pytest -nauto .\tests\glm --doctest-modules .\src\glum
+          environments: ${{ matrix.environment }}
+      - name: Install repository
+        run: pixi run -e ${{ matrix.environment }} postinstall
+      - name: Run pytest
+        run: pixi run -e ${{ matrix.environment }} test -nauto
       - name: Run doctest
-        if: matrix.os != 'windows-latest'
-        shell: bash -el {0}
         # Check that the readme example will work by running via doctest.
         # We run outside the repo to make the test a bit more similar to
         # a user running after installing with conda.
         run: |
           mkdir ../temp
           cp README.md ../temp
           cd ../temp
-          python -m doctest -v README.md
+          pixi run --manifest-path ../glum/pixi.toml -e ${{ matrix.environment }} python -m doctest -v README.md
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -9,62 +9,42 @@ on:
 
 jobs:
   linux-daily-unittests:
-    name: "Linux - daily unit tests - Python ${{ matrix.PYTHON_VERSION}} - ${{ matrix.NOTE }}"
+    name: "Linux - daily unit tests - ${{ matrix.NOTE }}"
     runs-on: ubuntu-latest
     env:
       CI: True
     strategy:
       fail-fast: false
       matrix:
         include:
-          - PYTHON_VERSION: '3.10'
+          - environment: 'nightly'
             NOTE: 'Nightly Builds' # run once with nightlies
-          - PYTHON_VERSION: '3.10'
-            NOTE: 'Default Builds' # run once with normal dependencies
+          - environment: 'py312'
+            NOTE: 'Python 3.12' # run once with normal dependencies
     steps:
       - name: Checkout branch
         uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
         with:
-          environment-file: environment.yml
-          create-args: >-
-            python=${{ matrix.PYTHON_VERSION }}
+          environments: ${{ matrix.environment }}
       - name: Install nightlies
         if: matrix.NOTE == 'Nightly Builds'
-        shell: bash -el {0}
-        run: |
-          # needed for tabmat
-          echo "Install compilation dependencies"
-          micromamba install -y c-compiler cxx-compiler 'cython!=3.0.4' jemalloc-local libgomp mako xsimd
-
-          PRE_WHEELS="https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/"
-          for pkg in pandas scikit-learn scipy; do
-            echo "Installing $pkg nightly"
-            micromamba remove -y --force $pkg
-            pip install --pre --no-deps --only-binary :all: --upgrade --timeout=60 -i $PRE_WHEELS $pkg
-          done
-          echo Install pyarrow nightly
-          micromamba remove -y --force pyarrow
-          pip install --extra-index-url https://pypi.fury.io/arrow-nightlies/ --prefer-binary --pre --no-deps pyarrow
-          echo Install tabmat nightly
-          micromamba remove -y --force tabmat
-          pip install --no-use-pep517 --no-deps git+https://github.com/Quantco/tabmat
+        run: pixi run -e ${{ matrix.environment }} install-nightlies
       - name: Install repository
-        shell: bash -el {0}
-        run: pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
+        run: pixi run -e ${{ matrix.environment }} postinstall
       - name: Run pytest
-        shell: bash -el {0}
-        run: pytest -nauto tests/glm --doctest-modules src/glum/
+        run: pixi run -e ${{ matrix.environment }} test -nauto
       - name: Run doctest
-        shell: bash -el {0}
         # Check that the readme example will work by running via doctest.
         # We run outside the repo to make the test a bit more similar to
         # a user running after installing with conda.
         run: |
           mkdir ../temp
           cp README.md ../temp
           cd ../temp
-          python -m doctest -v README.md
+          pixi run --manifest-path ../glum/pixi.toml -e ${{ matrix.environment }} python -m doctest -v README.md
+
       - name: Issue on failure
         uses: actions/github-script@v7
         if: ${{ failure() }}

diff --git a/.gitignore b/.gitignore
@@ -147,3 +147,7 @@ pkgs/*
 *.c
 *.cpp
 !*tmpl.cpp
+# pixi environments
+.pixi
+*.egg-info
+
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,24 +1,47 @@
+exclude: ^.pixi$
 repos:
-  - repo: https://github.com/Quantco/pre-commit-mirrors-ruff
-    rev: 0.4.7
+  - repo: local
     hooks:
-      - id: ruff-conda
-        exclude: ^src/glum_benchmarks/orig_sklearn_fork/
-      - id: ruff-format-conda
-        exclude: ^src/glum_benchmarks/orig_sklearn_fork/
-  - repo: https://github.com/Quantco/pre-commit-mirrors-mypy
-    rev: "1.10.0"
-    hooks:
-      - id: mypy-conda
-        args:
-          - --check-untyped-defs
-          - --ignore-missing-imports
-          - --namespace-packages
+      # ensure pixi environments are up to date
+      # workaround for https://github.com/prefix-dev/pixi/issues/1482
+      - id: pixi-install
+        name: pixi-install
+        entry: pixi install -e default -e lint
+        language: system
+        always_run: true
+        require_serial: true
+        pass_filenames: false
+      # ruff
+      - id: ruff
+        name: ruff
+        entry: pixi run -e lint ruff check --fix --exit-non-zero-on-fix --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: ruff-format
+        name: ruff-format
+        entry: pixi run -e lint ruff format --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      # cython-lint
+      - id: cython-lint
+        name: cython-lint
+        entry: pixi run -e lint cython-lint --no-pycodestyle
+        language: system
+        types: [cython]
+        require_serial: true
+      - id: cython-double-quote
+        name: cython-double-quote
+        entry: pixi run -e lint double-quote-cython-strings
+        language: system
+        types: [cython]
+        require_serial: true
+      # mypy
+      - id: mypy
+        name: mypy
+        entry: pixi run -e default mypy
         exclude: (^tests/|^src/glum_benchmarks/orig_sklearn_fork/)
-        additional_dependencies: [-c, conda-forge, types-setuptools=67.5, attrs]
-  - repo: https://github.com/Quantco/pre-commit-mirrors-cython-lint
-    rev: 0.16.2
-    hooks:
-      - id: cython-lint-conda
-        args: [--no-pycodestyle]
-      - id: double-quote-cython-strings-conda
+        language: system
+        types: [python]
+        require_serial: true
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -3,13 +3,11 @@ build:
   os: ubuntu-22.04
   tools:
     python: mambaforge-4.10
+  commands:
+    - mamba install -c conda-forge -c nodefaults pixi==0.22.0
+    - pixi run -e docs postinstall
+    - pixi run readthedocs
 sphinx:
   configuration: docs/conf.py
-python:
-  install:
-  - method: pip
-    path: .
-conda:
-  environment: environment.yml
 formats:
 - pdf
diff --git a/docs/contributing.rst b/docs/contributing.rst
@@ -31,39 +31,46 @@ Install for development
 --------------------------------------------------
 
 The first step is to set up a conda environment and install glum in editable mode.
-We strongly suggest to use ``mamba`` instead of ``conda`` as this provides the same functionality at much greater speed.
+The project uses [pixi](https://pixi.sh/latest/) for managing the environment. If you don't have pixi installed, start by [installing it](https://pixi.sh/latest/#installation).
 
 ::
 
-   # First, make sure you have conda-forge as your primary conda channel:
-   conda config --add channels conda-forge
-
    # Clone the repository
    git clone git@github.com:Quantco/glum.git
    cd glum
 
-   # Set up a conda environment with name "glum"
-   mamba env create
+   # Install the pre-commit hooks
+   pixi run pre-commit-install
+
+   # Install the dependencies, as well as and glum in editable mode
+   pixi run postinstall
 
    # If you want to install the dependencies necessary for benchmarking against other GLM packages:
-   mamba env update -n glum --file environment-benchmark.yml
+   pixi run -e benchmark postinstall
 
-   # If you want to work on the tutorial notebooks:
-   mamba env update -n glum --file environment-tutorial.yml
+   # If you want to work on the tutorial notebooks or the documentation:
+   pixi run -e docs postinstall
 
-   # Activate the previously created conda environment
-   conda activate glum
+   # You can run any command in the pixi environment with `pixi run <command>`. For example:
+   pixi run [-e ENVIRONMENT] ipython
 
-   # Set up our pre-commit hooks for black, mypy, isort and flake8.
-   pre-commit install
+   # Alternatively, you can create a shell with the pixi environment activated:
+   pixi shell
 
-   # Install this package in editable mode.
-   pip install --no-use-pep517 --disable-pip-version-check -e .
+   # Alternatively, a number of pixi tasks are available for commonly used commands.
+   # You can run them with `pixi run <task>`.
+   # To get a list of available tasks, run:
+   pixi task list
 
 
 Testing and continuous integration
 --------------------------------------------------
-The test suite is in ``tests/``. 
+The test suite is in ``tests/``. A pixi task is available to run the tests:
+
+::
+
+   pixi run test
+
 
 Golden master tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -80,12 +87,12 @@ If you want to skip the slow tests, add the ``-m "not slow"`` flag to any pytest
 Storing and modifying
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To store the golden master results:
+To store the golden master results, use the following pixi tasks:
 
 ::
 
-   python tests/glm/test_golden_master.py
-   python tests/glm/test_benchmark_golden_master.py
+   pixi run store-golden-master
+   pixi run store-benchmark-golden-master
 
 Add the ``--overwrite`` flag if you want to overwrite already existing golden master results
 
@@ -108,9 +115,7 @@ The documentation is built with a mix of Sphinx, autodoc, and nbsphinx. To devel
 
 ::
 
-   cd docs
-   make html
-   python -m http.server --directory _build/html
+   pixi run serve-docs
 
 Then, navigate to `<http://localhost:8000>`_ to view the documentation.
 

diff --git a/environment-benchmark.yml b/environment-benchmark.yml
diff --git a/environment-tutorial.yml b/environment-tutorial.yml