From c4f65d931f159ede1b4a4f3f5f7fbe71ab29a552 Mon Sep 17 00:00:00 2001 From: Alexis Fossart Date: Wed, 7 Jan 2026 09:51:08 +0100 Subject: [PATCH] fix: some issues with data output to postgresql --- cosmotech/coal/postgresql/runner.py | 4 ++-- cosmotech/coal/store/output/channel_spliter.py | 8 ++++++-- cosmotech/coal/store/output/postgres_channel.py | 1 + .../test_postgresql/test_postgresql_runner.py | 4 ++-- .../test_output/test_channel_spliter.py | 15 ++++++++------- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/cosmotech/coal/postgresql/runner.py b/cosmotech/coal/postgresql/runner.py index 15d74e9f..2cd8f2ad 100644 --- a/cosmotech/coal/postgresql/runner.py +++ b/cosmotech/coal/postgresql/runner.py @@ -71,13 +71,13 @@ def send_runner_metadata_to_postgresql( ( runner.get("id"), runner.get("name"), - runner.get("lastRunId"), + runner.get("lastRunInfo").get("lastRunId"), runner.get("runTemplateId"), ), ) conn.commit() LOGGER.info(T("coal.services.postgresql.metadata_updated")) - return runner.get("lastRunId") + return runner.get("lastRunInfo").get("lastRunId") def remove_runner_metadata_from_postgresql( diff --git a/cosmotech/coal/store/output/channel_spliter.py b/cosmotech/coal/store/output/channel_spliter.py index 671173ed..98d934d6 100644 --- a/cosmotech/coal/store/output/channel_spliter.py +++ b/cosmotech/coal/store/output/channel_spliter.py @@ -12,7 +12,7 @@ class ChannelSpliter(ChannelInterface): requirement_string: str = "(Requires any working interface)" - targets = [] + targets = list() available_interfaces: dict[str, ChannelInterface] = { "s3": AwsChannel, "az_storage": AzureStorageChannel, @@ -21,7 +21,7 @@ class ChannelSpliter(ChannelInterface): def __init__(self, dct: Dotdict = None): super().__init__(dct) - self.targets = [] + self.targets = list() if "outputs" not in self.configuration: raise AttributeError(T("coal.store.output.split.no_targets")) for output in self.configuration.outputs: @@ -45,6 +45,8 @@ def send(self, filter: Optional[list[str]] = None) -> bool: any_ok = i.send(filter=filter) or any_ok except Exception: LOGGER.error(T("coal.store.output.split.send.error").format(interface_name=i.__class__.__name__)) + if len(self.targets) < 2: + raise return any_ok def delete(self, filter: Optional[list[str]] = None) -> bool: @@ -54,4 +56,6 @@ def delete(self, filter: Optional[list[str]] = None) -> bool: any_ok = i.delete() or any_ok except Exception: LOGGER.error(T("coal.store.output.split.delete.error").format(interface_name=i.__class__.__name__)) + if len(self.targets) < 2: + raise return any_ok diff --git a/cosmotech/coal/store/output/postgres_channel.py b/cosmotech/coal/store/output/postgres_channel.py index e1f850fc..4298f774 100644 --- a/cosmotech/coal/store/output/postgres_channel.py +++ b/cosmotech/coal/store/output/postgres_channel.py @@ -30,6 +30,7 @@ def send(self, filter: Optional[list[str]] = None) -> bool: configuration=self.configuration, selected_tables=filter, fk_id=run_id, + replace=False, ) def delete(self): diff --git a/tests/unit/coal/test_postgresql/test_postgresql_runner.py b/tests/unit/coal/test_postgresql/test_postgresql_runner.py index 254fcc70..051ef445 100644 --- a/tests/unit/coal/test_postgresql/test_postgresql_runner.py +++ b/tests/unit/coal/test_postgresql/test_postgresql_runner.py @@ -33,7 +33,7 @@ def test_send_runner_metadata_to_postgresql(self, mock_connect, mock_postgres_ut mock_runner = { "id": "test-runner-id", "name": "Test Runner", - "lastRunId": "test-run-id", + "lastRunInfo": {"lastRunId": "test-run-id"}, "runTemplateId": "test-template-id", } @@ -87,7 +87,7 @@ def test_send_runner_metadata_to_postgresql(self, mock_connect, mock_postgres_ut assert upsert_call[0][1] == ( mock_runner["id"], mock_runner["name"], - mock_runner["lastRunId"], + mock_runner["lastRunInfo"]["lastRunId"], mock_runner["runTemplateId"], ) diff --git a/tests/unit/coal/test_store/test_output/test_channel_spliter.py b/tests/unit/coal/test_store/test_output/test_channel_spliter.py index e22755c3..2546ec92 100644 --- a/tests/unit/coal/test_store/test_output/test_channel_spliter.py +++ b/tests/unit/coal/test_store/test_output/test_channel_spliter.py @@ -244,10 +244,11 @@ def test_send_with_exception(self): # Act with patch.dict(ChannelSpliter.available_interfaces, {"s3": mock_channel_class}): spliter = ChannelSpliter(mock_config) - result = spliter.send() - # Assert - assert result is False + with pytest.raises(Exception): + result = spliter.send() + # Assert + assert result is False def test_delete_success(self): """Test delete method when all targets succeed.""" @@ -331,10 +332,10 @@ def test_delete_with_exception(self): # Act with patch.dict(ChannelSpliter.available_interfaces, {"s3": mock_channel_class}): spliter = ChannelSpliter(mock_config) - result = spliter.delete() - - # Assert - assert result is False + with pytest.raises(Exception): + result = spliter.delete() + # Assert + assert result is False def test_available_interfaces(self): """Test that available_interfaces are properly defined."""