diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index d9954d051..3c76be728 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -61,17 +61,32 @@ jobs: - name: Install library run: poetry install --no-interaction --all-extras #---------------------------------------------- - # run all tests with coverage + # run parallel tests with coverage #---------------------------------------------- - - name: Run all tests with coverage + - name: Run parallel tests with coverage continue-on-error: false run: | poetry run pytest tests/unit tests/e2e \ + -m "not serial" \ -n auto \ --cov=src \ --cov-report=xml \ --cov-report=term \ -v + + #---------------------------------------------- + # run serial tests with coverage + #---------------------------------------------- + - name: Run serial tests with coverage + continue-on-error: false + run: | + poetry run pytest tests/e2e \ + -m "serial" \ + --cov=src \ + --cov-append \ + --cov-report=xml \ + --cov-report=term \ + -v #---------------------------------------------- # check for coverage override diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b902e976..6be2dacaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Release History +# 4.2.2 (2025-12-01) +- Change default use_hybrid_disposition to False (databricks/databricks-sql-python#714 by @samikshya-db) +- Circuit breaker changes using pybreaker (databricks/databricks-sql-python#705 by @nikhilsuri-db) +- perf: Optimize telemetry latency logging to reduce overhead (databricks/databricks-sql-python#715 by @samikshya-db) +- basic e2e test for force telemetry verification (databricks/databricks-sql-python#708 by @nikhilsuri-db) +- Telemetry is ON by default to track connection stats. (Note : This strictly excludes PII, query text, and results) (databricks/databricks-sql-python#717 by @samikshya-db) + # 4.2.1 (2025-11-20) - Ignore transactions by default (databricks/databricks-sql-python#711 by @jayantsing-db) diff --git a/README.md b/README.md index ec82a3637..047515ba4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You are welcome to file an issue here for general use cases. You can also contac ## Requirements -Python 3.8 or above is required. +Python 3.9 or above is required. ## Documentation diff --git a/pyproject.toml b/pyproject.toml index 61c248e98..d2739c7d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databricks-sql-connector" -version = "4.2.1" +version = "4.2.2" description = "Databricks SQL Connector for Python" authors = ["Databricks "] license = "Apache-2.0" @@ -62,7 +62,10 @@ exclude = ['ttypes\.py$', 'TCLIService\.py$'] exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' [tool.pytest.ini_options] -markers = {"reviewed" = "Test case has been reviewed by Databricks"} +markers = [ + "reviewed: Test case has been reviewed by Databricks", + "serial: Tests that must run serially (not parallelized)" +] minversion = "6.0" log_cli = "false" log_cli_level = "INFO" diff --git a/src/databricks/sql/__init__.py b/src/databricks/sql/__init__.py index cd37e6ce1..7cf631e83 100644 --- a/src/databricks/sql/__init__.py +++ b/src/databricks/sql/__init__.py @@ -71,7 +71,7 @@ def __repr__(self): DATE = DBAPITypeObject("date") ROWID = DBAPITypeObject() -__version__ = "4.2.1" +__version__ = "4.2.2" USER_AGENT_NAME = "PyDatabricksSqlConnector" # These two functions are pyhive legacy diff --git a/src/databricks/sql/auth/common.py b/src/databricks/sql/auth/common.py index a764b036d..0e3a01918 100644 --- a/src/databricks/sql/auth/common.py +++ b/src/databricks/sql/auth/common.py @@ -51,7 +51,7 @@ def __init__( pool_connections: Optional[int] = None, pool_maxsize: Optional[int] = None, user_agent: Optional[str] = None, - telemetry_circuit_breaker_enabled: Optional[bool] = None, + telemetry_circuit_breaker_enabled: Optional[bool] = True, ): self.hostname = hostname self.access_token = access_token diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index c873700bc..1f17d54f2 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -328,7 +328,7 @@ def read(self) -> Optional[OAuthToken]: self.ignore_transactions = ignore_transactions self.force_enable_telemetry = kwargs.get("force_enable_telemetry", False) - self.enable_telemetry = kwargs.get("enable_telemetry", False) + self.enable_telemetry = kwargs.get("enable_telemetry", True) self.telemetry_enabled = TelemetryHelper.is_telemetry_enabled(self) TelemetryClientFactory.initialize_telemetry_client( diff --git a/tests/e2e/test_concurrent_telemetry.py b/tests/e2e/test_concurrent_telemetry.py index 546a2b8b2..bed348c2c 100644 --- a/tests/e2e/test_concurrent_telemetry.py +++ b/tests/e2e/test_concurrent_telemetry.py @@ -26,6 +26,7 @@ def run_in_threads(target, num_threads, pass_index=False): t.join() +@pytest.mark.serial class TestE2ETelemetry(PySQLPytestTestCase): @pytest.fixture(autouse=True) def telemetry_setup_teardown(self): diff --git a/tests/e2e/test_telemetry_e2e.py b/tests/e2e/test_telemetry_e2e.py index 917c8e5eb..0a57edd3c 100644 --- a/tests/e2e/test_telemetry_e2e.py +++ b/tests/e2e/test_telemetry_e2e.py @@ -43,8 +43,9 @@ def connection(self, extra_params=()): conn.close() +@pytest.mark.serial class TestTelemetryE2E(TelemetryTestBase): - """E2E tests for telemetry scenarios""" + """E2E tests for telemetry scenarios - must run serially due to shared host-level telemetry client""" @pytest.fixture(autouse=True) def telemetry_setup_teardown(self): @@ -58,6 +59,14 @@ def telemetry_setup_teardown(self): TelemetryClientFactory._stop_flush_thread() TelemetryClientFactory._initialized = False + # Clear feature flags cache to prevent state leakage between tests + from databricks.sql.common.feature_flag import FeatureFlagsContextFactory + with FeatureFlagsContextFactory._lock: + FeatureFlagsContextFactory._context_map.clear() + if FeatureFlagsContextFactory._executor: + FeatureFlagsContextFactory._executor.shutdown(wait=False) + FeatureFlagsContextFactory._executor = None + @pytest.fixture def telemetry_interceptors(self): """Setup reusable telemetry interceptors as a fixture""" @@ -142,7 +151,7 @@ def verify_events(self, captured_events, captured_futures, expected_count): else: assert len(captured_events) == expected_count, \ f"Expected {expected_count} events, got {len(captured_events)}" - + time.sleep(2) done, _ = wait(captured_futures, timeout=10) assert len(done) == expected_count, \ @@ -163,7 +172,7 @@ def verify_events(self, captured_events, captured_futures, expected_count): (True, False, 2, "enable_on_force_off"), (False, True, 2, "enable_off_force_on"), (False, False, 0, "both_off"), - (None, None, 0, "default_behavior"), + (None, None, 2, "default_behavior"), ]) def test_telemetry_flags(self, telemetry_interceptors, enable_telemetry, force_enable, expected_count, test_id): @@ -185,6 +194,8 @@ def test_telemetry_flags(self, telemetry_interceptors, enable_telemetry, cursor.execute("SELECT 1") cursor.fetchone() + # Give time for async telemetry submission after connection closes + time.sleep(0.5) self.verify_events(captured_events, captured_futures, expected_count) # Assert statement execution on latency event (if events exist)