test: improve some tests + retry others (#10239)

Author: anakin87

test/components/audio/test_whisper_remote.py

@@ -200,6 +200,7 @@ def test_whisper_remote_transcriber(self, test_files_path):
         not os.environ.get("OPENAI_API_KEY", None),
         reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
     )
+    @pytest.mark.flaky(reruns=3, reruns_delay=5)
     @pytest.mark.integration
     def test_whisper_remote_transcriber_pipeline_and_url_source(self):
         pipe = Pipeline()

test/components/connectors/test_openapi_connector.py

@@ -197,6 +197,7 @@ def test_serper_dev_integration(self):
         not os.environ.get("GITHUB_TOKEN", None), reason="Export an env var called GITHUB_TOKEN to run this test."
     )
     @pytest.mark.integration
+    @pytest.mark.flaky(reruns=3, reruns_delay=5)
     def test_github_api_integration(self):
         component = OpenAPIConnector(
             openapi_spec="https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.json",

test/components/fetchers/test_link_content_fetcher.py

@@ -182,7 +182,10 @@ def test_request_headers_merging_and_ua_override(self):
             assert sent_headers["Accept-Language"] == "fr-FR"
             assert sent_headers["User-Agent"] == "ua-sync-1"  # rotating UA wins
 
-    @pytest.mark.integration
+
+@pytest.mark.flaky(reruns=3, reruns_delay=5)
+@pytest.mark.integration
+class TestLinkContentFetcherIntegration:
     def test_link_content_fetcher_html(self):
         """
         Test fetching HTML content from a real URL.
@@ -195,7 +198,6 @@ def test_link_content_fetcher_html(self):
         assert "url" in first_stream.meta and first_stream.meta["url"] == HTML_URL
         assert first_stream.mime_type == "text/html"
 
-    @pytest.mark.integration
     def test_link_content_fetcher_text(self):
         """
         Test fetching text content from a real URL.
@@ -208,7 +210,6 @@ def test_link_content_fetcher_text(self):
         assert "url" in first_stream.meta and first_stream.meta["url"] == TEXT_URL
         assert first_stream.mime_type == "text/plain"
 
-    @pytest.mark.integration
     def test_link_content_fetcher_multiple_different_content_types(self):
         """
         This test is to ensure that the fetcher can handle a list of URLs that contain different content types.
@@ -225,7 +226,6 @@ def test_link_content_fetcher_multiple_different_content_types(self):
                 assert len(stream.data) > 0
                 assert stream.mime_type == "application/pdf"
 
-    @pytest.mark.integration
     def test_link_content_fetcher_multiple_html_streams(self):
         """
         This test is to ensure that the fetcher can handle a list of URLs that contain different content types,
@@ -244,7 +244,6 @@ def test_link_content_fetcher_multiple_html_streams(self):
                 assert len(stream.data) > 0
                 assert stream.mime_type == "application/pdf"
 
-    @pytest.mark.integration
     def test_mix_of_good_and_failed_requests(self):
         """
         This test is to ensure that the fetcher can handle a list of URLs that contain URLs that fail to be fetched.
@@ -259,8 +258,8 @@ def test_mix_of_good_and_failed_requests(self):
         assert first_stream.mime_type == "text/html"
 
 
+@pytest.mark.asyncio
 class TestLinkContentFetcherAsync:
-    @pytest.mark.asyncio
     async def test_run_async(self):
         """Test basic async fetching with a mocked response"""
         with patch("haystack.components.fetchers.link_content.httpx.AsyncClient.get") as mock_get:
@@ -276,7 +275,6 @@ async def test_run_async(self):
             assert first_stream.meta["content_type"] == "text/plain"
             assert first_stream.mime_type == "text/plain"
 
-    @pytest.mark.asyncio
     async def test_run_async_multiple(self):
         """Test async fetching of multiple URLs with mocked responses"""
         with patch("haystack.components.fetchers.link_content.httpx.AsyncClient.get") as mock_get:
@@ -295,14 +293,12 @@ async def test_run_async_multiple(self):
                 assert stream.meta["content_type"] == "text/plain"
                 assert stream.mime_type == "text/plain"
 
-    @pytest.mark.asyncio
     async def test_run_async_empty_urls(self):
         """Test async fetching with empty URL list"""
         fetcher = LinkContentFetcher()
         streams = (await fetcher.run_async(urls=[]))["streams"]
         assert len(streams) == 0
 
-    @pytest.mark.asyncio
     async def test_run_async_error_handling(self):
         """Test error handling for async fetching"""
         with patch("haystack.components.fetchers.link_content.httpx.AsyncClient.get") as mock_get:
@@ -322,7 +318,6 @@ async def test_run_async_error_handling(self):
             with pytest.raises(httpx.HTTPStatusError):
                 await fetcher.run_async(urls=["https://www.example.com"])
 
-    @pytest.mark.asyncio
     async def test_run_async_user_agent_rotation(self):
         """Test user agent rotation in async fetching"""
         with (
@@ -355,34 +350,6 @@ async def test_run_async_user_agent_rotation(self):
 
             mock_sleep.assert_called_once()
 
-    @pytest.mark.asyncio
-    @pytest.mark.integration
-    async def test_run_async_multiple_integration(self):
-        """Test async fetching of multiple URLs with real HTTP requests"""
-        fetcher = LinkContentFetcher()
-        streams = (await fetcher.run_async([HTML_URL, TEXT_URL]))["streams"]
-        assert len(streams) == 2
-
-        for stream in streams:
-            assert "Haystack" in stream.data.decode("utf-8")
-
-            if stream.meta["url"] == HTML_URL:
-                assert stream.meta["content_type"] == "text/html"
-                assert stream.mime_type == "text/html"
-            elif stream.meta["url"] == TEXT_URL:
-                assert stream.meta["content_type"] == "text/plain"
-                assert stream.mime_type == "text/plain"
-
-    @pytest.mark.asyncio
-    @pytest.mark.integration
-    async def test_run_async_with_client_kwargs(self):
-        """Test async fetching with custom client kwargs"""
-        fetcher = LinkContentFetcher(client_kwargs={"follow_redirects": True, "timeout": 10.0})
-        streams = (await fetcher.run_async([HTML_URL]))["streams"]
-        assert len(streams) == 1
-        assert "Haystack" in streams[0].data.decode("utf-8")
-
-    @pytest.mark.asyncio
     async def test_request_headers_merging_and_ua_override(self):
         # Patch the AsyncClient class to control the instance created by LinkContentFetcher
         with patch("haystack.components.fetchers.link_content.httpx.AsyncClient") as AsyncClientMock:
@@ -405,7 +372,6 @@ async def test_request_headers_merging_and_ua_override(self):
             assert sent_headers["Accept-Language"] == "de-DE"
             assert sent_headers["User-Agent"] == "ua-async-1"  # rotating UA wins
 
-    @pytest.mark.asyncio
     async def test_duplicated_request_headers_merging(self):
         # Patch the AsyncClient class to control the instance created by LinkContentFetcher
         with patch("haystack.components.fetchers.link_content.httpx.AsyncClient") as AsyncClientMock:
@@ -439,3 +405,31 @@ async def test_duplicated_request_headers_merging(self):
 
             assert "x-test-header" in existing_keys
             assert existing_keys["x-test-header"] == "X-TeSt-HeAdEr"
+
+
+@pytest.mark.flaky(reruns=3, reruns_delay=5)
+@pytest.mark.integration
+@pytest.mark.asyncio
+class TestLinkContentFetcherAsyncIntegration:
+    async def test_run_async_multiple_integration(self):
+        """Test async fetching of multiple URLs with real HTTP requests"""
+        fetcher = LinkContentFetcher()
+        streams = (await fetcher.run_async([HTML_URL, TEXT_URL]))["streams"]
+        assert len(streams) == 2
+
+        for stream in streams:
+            assert "Haystack" in stream.data.decode("utf-8")
+
+            if stream.meta["url"] == HTML_URL:
+                assert stream.meta["content_type"] == "text/html"
+                assert stream.mime_type == "text/html"
+            elif stream.meta["url"] == TEXT_URL:
+                assert stream.meta["content_type"] == "text/plain"
+                assert stream.mime_type == "text/plain"
+
+    async def test_run_async_with_client_kwargs(self):
+        """Test async fetching with custom client kwargs"""
+        fetcher = LinkContentFetcher(client_kwargs={"follow_redirects": True, "timeout": 10.0})
+        streams = (await fetcher.run_async([HTML_URL]))["streams"]
+        assert len(streams) == 1
+        assert "Haystack" in streams[0].data.decode("utf-8")

test/components/generators/chat/test_openai.py

@@ -1088,8 +1088,8 @@ def test_live_run_with_toolset(self, tools):
         tool_call = message.tool_call
         assert isinstance(tool_call, ToolCall)
         assert tool_call.tool_name == "weather"
-        assert tool_call.arguments == {"city": "Paris"}
-        assert message.meta["finish_reason"] == "tool_calls"
+        assert tool_call.arguments.keys() == {"city"}
+        assert "Paris" in tool_call.arguments["city"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),

test/components/generators/chat/test_openai_async.py

@@ -344,7 +344,7 @@ async def test_callback(chunk: StreamingChunk):
     @pytest.mark.integration
     @pytest.mark.asyncio
     async def test_run_async_cancellation_integration(self):
-        generator = OpenAIChatGenerator(model="gpt-4")
+        generator = OpenAIChatGenerator(model="gpt-4.1-nano")
         messages = [ChatMessage.from_user("Write me an essay about the history of jazz music, at least 500 words.")]
         received_chunks = []
 

test/components/generators/chat/test_openai_responses.py

@@ -273,6 +273,15 @@ def weather_function(city: str) -> dict[str, Any]:
     return weather_info.get(city, {"weather": "unknown", "temperature": 0, "unit": "celsius"})
 
 
+# Tool Function used in the test_live_run_with_agent_streaming_and_reasoning test
+def calculate(expression: str) -> dict:
+    try:
+        result = eval(expression, {"__builtins__": {}})
+        return {"result": result}
+    except Exception as e:
+        return {"error": str(e)}
+
+
 @pytest.fixture
 def tools():
     weather_tool = Tool(
@@ -1064,7 +1073,8 @@ def test_live_run_with_toolset(self, tools):
         tool_call = message.tool_call
         assert isinstance(tool_call, ToolCall)
         assert tool_call.tool_name == "weather"
-        assert tool_call.arguments == {"city": "Paris"}
+        assert tool_call.arguments.keys() == {"city"}
+        assert "Paris" in tool_call.arguments["city"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),
@@ -1169,14 +1179,6 @@ def test_live_run_with_tools_streaming_and_reasoning(self, tools):
     )
     @pytest.mark.integration
     def test_live_run_with_agent_streaming_and_reasoning(self):
-        # Tool Function
-        def calculate(expression: str) -> dict:
-            try:
-                result = eval(expression, {"__builtins__": {}})
-                return {"result": result}
-            except Exception as e:
-                return {"error": str(e)}
-
         # Tool Definition
         calculator_tool = Tool(
             name="calculator",