Develop -> Main

.github/workflows/test-pypi-upload.yml

@@ -0,0 +1,69 @@
+name: Upload to TestPyPI
+
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch: # Allow manual triggers for testing
+
+jobs:
+  build-and-upload:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Update version with .devN format
+        id: version
+        run: |
+          python scripts/update_version_for_testpypi.py
+          VERSION=$(grep -Po '^version\s*=\s*"\K[^"]+' pyproject.toml)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Building version: $VERSION"
+
+      - name: Build package
+        run: |
+          python -m build
+          echo "Build complete. Package contents:"
+          ls -lh dist/
+
+      - name: Check package metadata
+        run: |
+          twine check dist/*
+
+      - name: Upload to TestPyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_UPLOAD_TOKEN }}
+          TWINE_REPOSITORY: testpypi
+        run: |
+          twine upload --verbose dist/*
+
+      - name: Display installation instructions
+        if: success()
+        run: |
+          echo "════════════════════════════════════════════════════════════════"
+          echo "✅ Package uploaded successfully to TestPyPI!"
+          echo "════════════════════════════════════════════════════════════════"
+          echo ""
+          echo "Version: ${{ steps.version.outputs.version }}"
+          echo ""
+          echo "Test installation with:"
+          echo "  pip install --index-url https://test.pypi.org/simple/ \\"
+          echo "              --extra-index-url https://pypi.org/simple \\"
+          echo "              scrython==${{ steps.version.outputs.version }}"
+          echo ""
+          echo "View on TestPyPI: https://test.pypi.org/project/scrython/"
+          echo "════════════════════════════════════════════════════════════════"

scripts/update_version_for_testpypi.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Update version in pyproject.toml with dev identifier for TestPyPI uploads."""
+
+import os
+import re
+import sys
+from pathlib import Path
+
+
+def get_unique_identifier() -> str:
+    """Get a unique identifier for the dev version.
+
+    Uses GitHub Actions run number if available, otherwise falls back to
+    a timestamp. This ensures each upload has a unique, incrementing version.
+    """
+    # GitHub Actions provides a unique, incrementing run number
+    run_number = os.environ.get("GITHUB_RUN_NUMBER")
+    if run_number:
+        return run_number
+
+    # Fallback for local testing: use timestamp
+    from datetime import datetime, timezone
+    return datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
+
+
+def update_version_in_pyproject() -> str:
+    """Add dev identifier to version in pyproject.toml."""
+    pyproject_path = Path("pyproject.toml")
+
+    if not pyproject_path.exists():
+        print("ERROR: pyproject.toml not found in current directory")
+        sys.exit(1)
+
+    content = pyproject_path.read_text()
+
+    # Find current version
+    version_match = re.search(r'^version\s*=\s*"([^"]+)"', content, re.MULTILINE)
+    if not version_match:
+        print("ERROR: Could not find version in pyproject.toml")
+        sys.exit(1)
+
+    current_version = version_match.group(1)
+    unique_id = get_unique_identifier()
+
+    # Create new version with PEP 440 dev identifier (accepted by PyPI/TestPyPI)
+    new_version = f"{current_version}.dev{unique_id}"
+
+    # Replace version in content
+    new_content = re.sub(
+        r'^version\s*=\s*"[^"]+"',
+        f'version = "{new_version}"',
+        content,
+        count=1,
+        flags=re.MULTILINE
+    )
+
+    # Write back (temporary, only in workflow workspace)
+    pyproject_path.write_text(new_content)
+
+    print(f"Version updated: {current_version} -> {new_version}")
+    return new_version
+
+
+if __name__ == "__main__":
+    try:
+        version = update_version_in_pyproject()
+        # Output for GitHub Actions
+        print(f"::set-output name=version::{version}")
+        sys.exit(0)
+    except Exception as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        sys.exit(1)

scrython/bulk_data/bulk_data_mixins.py

@@ -1,7 +1,9 @@
 import gzip
 import json
 from typing import Any
-from urllib.request import urlopen
+from urllib.request import Request, urlopen
+
+from ..base import ScrythonRequestHandler
 
 
 class BulkDataObjectMixin:
@@ -161,18 +163,22 @@ def download(
         """
         download_url = self.download_uri
 
+        request = Request(download_url)
+        request.add_header("User-Agent", ScrythonRequestHandler._user_agent)
+        request.add_header("Accept-Encoding", "gzip, identity")
+
         # Optional progress bar
         if progress:
             try:
-                from tqdm import tqdm
+                from tqdm.auto import tqdm
             except ImportError as exc:
                 raise ImportError(
                     "tqdm is required for progress bars. "
                     "Install with: pip install scrython[progress] or pip install tqdm"
                 ) from exc
 
             # Download with progress bar
-            with urlopen(download_url) as response:
+            with urlopen(request) as response:
                 # Check actual HTTP Content-Encoding header
                 content_encoding = response.info().get("Content-Encoding", "").lower()
 
@@ -202,7 +208,7 @@ def download(
                 data = downloaded_data
         else:
             # Download without progress bar
-            with urlopen(download_url) as response:
+            with urlopen(request) as response:
                 # Check actual HTTP Content-Encoding header
                 content_encoding = response.info().get("Content-Encoding", "").lower()
 

tests/test_bulk_data.py

@@ -312,3 +312,32 @@ def test_download_uncompressed_with_progress(self, mock_urlopen):
             assert result == test_data
             assert len(result) == 1
             assert result[0]["name"] == "Test Card"
+
+    def test_download_sets_headers(self, mock_urlopen):
+        """Test download sets proper User-Agent and Accept-Encoding headers."""
+        from urllib.request import Request
+
+        from scrython.base import ScrythonRequestHandler
+
+        mock_urlopen.set_response("bulk_data/by_id.json")
+        bulk = ByType(type="oracle_cards")
+
+        with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download:
+            # Set up mock to allow inspection of the Request object
+            mock_response = MagicMock()
+            mock_response.read.return_value = b"[]"
+            mock_response.info.return_value.get.return_value = ""
+            mock_response.__enter__.return_value = mock_response
+            mock_response.__exit__.return_value = None
+            mock_download.return_value = mock_response
+
+            bulk.download()
+
+            # Verify urlopen was called with a Request object
+            mock_download.assert_called_once()
+            request = mock_download.call_args[0][0]
+            assert isinstance(request, Request)
+
+            # Verify headers are set correctly
+            assert request.get_header("User-agent") == ScrythonRequestHandler._user_agent
+            assert request.get_header("Accept-encoding") == "gzip, identity"