REFACTOR: remove bundled ODBC driver from mssql-python wheel; require mssql-python-odbc (Phase 2)

.github/copilot-instructions.md

@@ -15,7 +15,7 @@ Paths you will touch:
 
 - `mssql_python/pybind/CMakeLists.txt` — all platform/architecture build conditionals live here.
 - `mssql_python/pybind/build.sh` / `build.bat` — build entry points; `configure_dylibs.sh` fixes macOS dylib paths.
-- `mssql_python/libs/{windows,macos,linux}/...` — prebuilt ODBC binaries. **NEVER hand-edit these.**
+- `mssql_python_odbc/libs/{windows,macos,linux}/...` — prebuilt ODBC binaries (moved out of the mssql-python wheel into the standalone `mssql-python-odbc` package in Phase 2). **NEVER hand-edit these.**
 - `setup.py` / `pyproject.toml` — packaging and wheel/platform tagging.
 - `mssql_python/mssql_python.pyi` + `mssql_python/py.typed` — PEP 561 type stubs. Update stubs when the public API changes.
 - `tests/` — mostly-numbered `test_NNN_*.py` files (a live-SQL-Server integration suite plus no-DB dependency checks).

.github/prompts/create-pr.prompt.md

@@ -146,15 +146,15 @@ git status
 > ⚠️ **AVOID:** `git add .` stages everything including binary files. Always stage specific files.
 
 **Files to typically EXCLUDE from commits:**
-- `mssql_python/libs/**/*.dylib` - macOS libraries
-- `mssql_python/libs/**/*.so` - Linux libraries  
+- `mssql_python_odbc/libs/**/*.dylib` - macOS libraries
+- `mssql_python_odbc/libs/**/*.so` - Linux libraries  
 - `mssql_python/*.so` or `*.pyd` - Built extensions
 - `*.dll` - Windows libraries
 - Virtual environments (`myvenv/`, `testenv/`, etc.)
 
 **To unstage accidentally added binary files:**
 ```bash
-git restore --staged mssql_python/libs/
+git restore --staged mssql_python_odbc/libs/
 git restore --staged "*.dylib" "*.so" "*.pyd"
 ```
 

.gitignore

@@ -23,11 +23,10 @@ build/
 **/build/
 mssql_python.egg-info/
 
-# ODBC package (mssql-python-odbc): transition-period build artifacts.
-# During Phase 2, setup_odbc.py copies the current platform's driver binaries
-# from mssql_python/libs into this tree so a wheel can be built locally. The
-# release pipeline populates these per-platform; do not commit the copies.
-mssql_python_odbc/libs/
+# ODBC package (mssql-python-odbc).
+# The driver binaries under mssql_python_odbc/libs/ are the committed source of
+# truth for the standalone package (moved here from mssql_python/libs in Phase 2).
+# The release pipeline still overwrites them per-platform for the full matrix.
 mssql_python_odbc.egg-info/
 
 # Python bytecode

OneBranchPipelines/stages/build-linux-single-stage.yml

@@ -23,10 +23,20 @@ parameters:
   - name: oneBranchType
     type: string
     default: 'Official'
+  # Phase 2: stages this build depends on (e.g. [ConsolidateOdbc]) so the external
+  # mssql-python-odbc wheel exists before pytest. Default [] = no dependency.
+  - name: odbcDependsOn
+    type: object
+    default: []
+  # Phase 2: install the external ODBC wheel (from ConsolidateOdbc) before pytest.
+  - name: installOdbcWheel
+    type: boolean
+    default: false
 
 stages:
   - stage: ${{ parameters.stageName }}
     displayName: 'Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}'
+    dependsOn: ${{ parameters.odbcDependsOn }}
     jobs:
       - job: ${{ parameters.jobName }}
         displayName: 'Build Wheels - ${{ parameters.linuxTag }} ${{ parameters.arch }}'
@@ -191,6 +201,33 @@ stages:
             env:
               DB_PASSWORD: $(DB_PASSWORD)
 
+          # =========================
+          # PHASE 2: EXTERNAL ODBC DRIVER PACKAGE
+          # =========================
+          # Phase 2 removed the bundled ODBC driver from the mssql-python wheel, so the
+          # isolated-install pytest below must obtain the driver from the external
+          # mssql-python-odbc package. Download the wheels built earlier this run
+          # (ConsolidateOdbc) into the repo dir (bind-mounted at /workspace in the build
+          # container) and flatten them so PIP_FIND_LINKS=/workspace/odbc_wheels lets the
+          # in-container `pip install <mssql_python wheel>` resolve the pinned
+          # mssql-python-odbc==18.6.2 dependency locally (no PyPI needed for it).
+          - ${{ if parameters.installOdbcWheel }}:
+            - task: DownloadPipelineArtifact@2
+              displayName: 'Download consolidated ODBC wheels'
+              inputs:
+                buildType: 'current'
+                artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts'
+                itemPattern: '**/*.whl'
+                targetPath: '$(Build.SourcesDirectory)/odbc_wheels'
+            - bash: |
+                set -euo pipefail
+                # Flatten all downloaded wheels into the find-links root so pip (via
+                # PIP_FIND_LINKS) sees them without recursion.
+                find "$(Build.SourcesDirectory)/odbc_wheels" -name '*.whl' -exec mv -f {} "$(Build.SourcesDirectory)/odbc_wheels/" \; || true
+                echo "ODBC wheels available for find-links:"
+                ls -1 "$(Build.SourcesDirectory)/odbc_wheels"/*.whl
+              displayName: 'Flatten ODBC wheels for pip find-links'
+
           # Build wheels for all Python versions (3.10-3.14) and test each one
           - script: |
               set -euxo pipefail
@@ -206,7 +243,7 @@ stages:
 
                 if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then
                   # Manylinux (glibc-based) - use bash
-                  docker exec -e PYBIN=$PYBIN -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" -e MANYLINUX_TAG="$(LINUX_TAG)" build-$(LINUX_TAG)-$(ARCH) bash -lc '
+                  docker exec -e PYBIN=$PYBIN -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" -e MANYLINUX_TAG="$(LINUX_TAG)" -e PIP_FIND_LINKS="/workspace/odbc_wheels" build-$(LINUX_TAG)-$(ARCH) bash -lc '
                     set -euxo pipefail;
 
                     # Step 1: Setup Python environment
@@ -274,7 +311,7 @@ stages:
                   '
                 else
                   # Musllinux (musl libc-based) - use sh
-                  docker exec -e PYBIN=$PYBIN -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" -e MANYLINUX_TAG="$(LINUX_TAG)" build-$(LINUX_TAG)-$(ARCH) sh -lc '
+                  docker exec -e PYBIN=$PYBIN -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" -e MANYLINUX_TAG="$(LINUX_TAG)" -e PIP_FIND_LINKS="/workspace/odbc_wheels" build-$(LINUX_TAG)-$(ARCH) sh -lc '
                     set -euxo pipefail;
 
                     # Step 1: Setup Python environment

OneBranchPipelines/stages/build-macos-single-stage.yml

@@ -20,10 +20,20 @@ parameters:
   - name: oneBranchType
     type: string
     default: 'Official'
+  # Phase 2: stages this build depends on (e.g. [ConsolidateOdbc]) so the external
+  # mssql-python-odbc wheel exists before pytest. Default [] = no dependency.
+  - name: odbcDependsOn
+    type: object
+    default: []
+  # Phase 2: install the external ODBC wheel (from ConsolidateOdbc) before pytest.
+  - name: installOdbcWheel
+    type: boolean
+    default: false
 
 stages:
   - stage: ${{ parameters.stageName }}
     displayName: 'macOS Py${{ parameters.pythonVersion }} Universal2'
+    dependsOn: ${{ parameters.odbcDependsOn }}
     jobs:
       - job: ${{ parameters.jobName }}
         displayName: 'Build Wheel - Py${{ parameters.pythonVersion }} Universal2'
@@ -176,6 +186,37 @@ stages:
               bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh
             displayName: 'Install mssql_py_core from NuGet'
 
+          # =========================
+          # PHASE 2: EXTERNAL ODBC DRIVER PACKAGE
+          # =========================
+          # Phase 2 removed the bundled ODBC driver from the mssql-python wheel, so the
+          # driver is resolved from the separate mssql-python-odbc package. Install the
+          # matching wheel built earlier in this run (ConsolidateOdbc) before pytest.
+          - ${{ if parameters.installOdbcWheel }}:
+            - task: DownloadPipelineArtifact@2
+              displayName: 'Download consolidated ODBC wheels'
+              inputs:
+                buildType: 'current'
+                artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts'
+                itemPattern: '**/*.whl'
+                targetPath: '$(Pipeline.Workspace)/odbc_wheels'
+            - script: |
+                set -e
+                LINKS=$(dirname "$(find "$(Pipeline.Workspace)/odbc_wheels" -name '*.whl' | head -1)")
+                if [ -z "$LINKS" ]; then echo "No ODBC wheel found in downloaded artifact" >&2; exit 1; fi
+                echo "Installing mssql-python-odbc from $LINKS"
+                python -m pip install --no-index --find-links "$LINKS" mssql-python-odbc
+                # The repo checkout ships a committed mssql_python_odbc/ dir whose libs/
+                # would SHADOW the just-installed site-packages wheel when pytest runs from
+                # the checkout root. Remove it so pytest resolves the driver from the
+                # EXTERNAL wheel (mirrors the Linux isolated-install job); otherwise a
+                # broken/no-op install would go undetected. setup.py excludes
+                # mssql_python_odbc, so the later mssql-python bdist_wheel is unaffected.
+                rm -rf "$(Build.SourcesDirectory)/mssql_python_odbc"
+                # Assert the driver package now resolves to site-packages, NOT the checkout.
+                python -c "import os,mssql_python_odbc as m; p=os.path.realpath(os.path.dirname(m.__file__)); s=os.path.realpath(os.environ['BUILD_SOURCESDIRECTORY']); assert not p.startswith(s), 'mssql_python_odbc resolved inside checkout: '+p; print('mssql_python_odbc resolved from installed wheel:', p)"
+              displayName: 'Install external ODBC wheel (mssql-python-odbc)'
+
           # =========================
           # TESTING
           # =========================