From 7b1b718123afd80c0f68212946e4179bcd6db67f Mon Sep 17 00:00:00 2001 From: shollyman Date: Sat, 13 Sep 2025 08:02:19 -0700 Subject: [PATCH 1/3] feat: add additional query stats (#2270) * feat: add additional query stats This PR adds support for incremental query stats. --- google/cloud/bigquery/job/__init__.py | 2 + google/cloud/bigquery/job/query.py | 67 +++++++++++++++++++++++++++ tests/unit/job/test_query.py | 17 +++++++ tests/unit/job/test_query_stats.py | 61 ++++++++++++++++++++++++ 4 files changed, 147 insertions(+) diff --git a/google/cloud/bigquery/job/__init__.py b/google/cloud/bigquery/job/__init__.py index f51311b0b..4cda65965 100644 --- a/google/cloud/bigquery/job/__init__.py +++ b/google/cloud/bigquery/job/__init__.py @@ -39,6 +39,7 @@ from google.cloud.bigquery.job.query import QueryPlanEntryStep from google.cloud.bigquery.job.query import ScriptOptions from google.cloud.bigquery.job.query import TimelineEntry +from google.cloud.bigquery.job.query import IncrementalResultStats from google.cloud.bigquery.enums import Compression from google.cloud.bigquery.enums import CreateDisposition from google.cloud.bigquery.enums import DestinationFormat @@ -84,4 +85,5 @@ "SourceFormat", "TransactionInfo", "WriteDisposition", + "IncrementalResultStats", ] diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index b377f979d..38b8a7148 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -197,6 +197,66 @@ def from_api_repr(cls, stats: Dict[str, str]) -> "DmlStats": return cls(*args) +class IncrementalResultStats: + """IncrementalResultStats provides information about incremental query execution.""" + + def __init__(self): + self._properties = {} + + @classmethod + def from_api_repr(cls, resource) -> "IncrementalResultStats": + """Factory: construct instance from the JSON repr. + + Args: + resource(Dict[str: object]): + IncrementalResultStats representation returned from API. + + Returns: + google.cloud.bigquery.job.IncrementalResultStats: + stats parsed from ``resource``. + """ + entry = cls() + entry._properties = resource + return entry + + @property + def disabled_reason(self): + """Optional[string]: Reason why incremental results were not + written by the query. + """ + return _helpers._str_or_none(self._properties.get("disabledReason")) + + @property + def result_set_last_replace_time(self): + """Optional[datetime]: The time at which the result table's contents + were completely replaced. May be absent if no results have been written + or the query has completed.""" + from google.cloud._helpers import _rfc3339_nanos_to_datetime + + value = self._properties.get("resultSetLastReplaceTime") + if value: + try: + return _rfc3339_nanos_to_datetime(value) + except ValueError: + pass + return None + + @property + def result_set_last_modify_time(self): + """Optional[datetime]: The time at which the result table's contents + were modified. May be absent if no results have been written or the + query has completed.""" + from google.cloud._helpers import _rfc3339_nanos_to_datetime + + value = self._properties.get("resultSetLastModifyTime") + if value: + try: + return _rfc3339_nanos_to_datetime(value) + except ValueError: + pass + return None + + class IndexUnusedReason(typing.NamedTuple): """Reason about why no search index was used in the search query (or sub-query). @@ -1339,6 +1399,13 @@ def bi_engine_stats(self) -> Optional[BiEngineStats]: else: return BiEngineStats.from_api_repr(stats) + @property + def incremental_result_stats(self) -> Optional[IncrementalResultStats]: + stats = self._job_statistics().get("incrementalResultStats") + if stats is None: + return None + return IncrementalResultStats.from_api_repr(stats) + def _blocking_poll(self, timeout=None, **kwargs): self._done_timeout = timeout self._transport_timeout = timeout diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index ef6429598..4a6771c46 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -838,6 +838,23 @@ def test_search_stats(self): assert isinstance(job.search_stats, SearchStats) assert job.search_stats.mode == "INDEX_USAGE_MODE_UNSPECIFIED" + def test_incremental_result_stats(self): + from google.cloud.bigquery.job.query import IncrementalResultStats + + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, self.QUERY, client) + assert job.incremental_result_stats is None + + statistics = job._properties["statistics"] = {} + assert job.incremental_result_stats is None + + query_stats = statistics["query"] = {} + assert job.incremental_result_stats is None + + query_stats["incrementalResultStats"] = {"disabledReason": "BAZ"} + assert isinstance(job.incremental_result_stats, IncrementalResultStats) + assert job.incremental_result_stats.disabled_reason == "BAZ" + def test_reload_query_results_uses_transport_timeout(self): conn = make_connection({}) client = _make_client(self.PROJECT, connection=conn) diff --git a/tests/unit/job/test_query_stats.py b/tests/unit/job/test_query_stats.py index 61b278d43..c7c7a31e0 100644 --- a/tests/unit/job/test_query_stats.py +++ b/tests/unit/job/test_query_stats.py @@ -13,6 +13,7 @@ # limitations under the License. from .helpers import _Base +import datetime class TestBiEngineStats: @@ -520,3 +521,63 @@ def test_from_api_repr_normal(self): self.assertEqual(entry.pending_units, self.PENDING_UNITS) self.assertEqual(entry.completed_units, self.COMPLETED_UNITS) self.assertEqual(entry.slot_millis, self.SLOT_MILLIS) + + +class TestIncrementalResultStats: + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.job import IncrementalResultStats + + return IncrementalResultStats + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor_defaults(self): + stats = self._make_one() + assert stats.disabled_reason is None + assert stats.result_set_last_replace_time is None + assert stats.result_set_last_modify_time is None + + def test_from_api_repr_partial_stats(self): + klass = self._get_target_class() + stats = klass.from_api_repr({"disabledReason": "FOO"}) + + assert isinstance(stats, klass) + assert stats.disabled_reason == "FOO" + assert stats.result_set_last_replace_time is None + assert stats.result_set_last_modify_time is None + + def test_from_api_repr_full_stats(self): + klass = self._get_target_class() + stats = klass.from_api_repr( + { + "disabledReason": "BAR", + "resultSetLastReplaceTime": "2025-01-02T03:04:05.06Z", + "resultSetLastModifyTime": "2025-02-02T02:02:02.02Z", + } + ) + + assert isinstance(stats, klass) + assert stats.disabled_reason == "BAR" + assert stats.result_set_last_replace_time == datetime.datetime( + 2025, 1, 2, 3, 4, 5, 60000, tzinfo=datetime.timezone.utc + ) + assert stats.result_set_last_modify_time == datetime.datetime( + 2025, 2, 2, 2, 2, 2, 20000, tzinfo=datetime.timezone.utc + ) + + def test_from_api_repr_invalid_stats(self): + klass = self._get_target_class() + stats = klass.from_api_repr( + { + "disabledReason": "BAR", + "resultSetLastReplaceTime": "xxx", + "resultSetLastModifyTime": "yyy", + } + ) + + assert isinstance(stats, klass) + assert stats.disabled_reason == "BAR" + assert stats.result_set_last_replace_time is None + assert stats.result_set_last_modify_time is None From c9aba64c1f7240f1ad2caa00d55a1a4f86bdc8a3 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 15 Sep 2025 14:21:21 +0200 Subject: [PATCH 2/3] chore(deps): update all dependencies (#2275) --- samples/desktopapp/requirements-test.txt | 2 +- samples/desktopapp/requirements.txt | 2 +- samples/geography/requirements-test.txt | 2 +- samples/geography/requirements.txt | 10 +++++----- samples/magics/requirements-test.txt | 2 +- samples/magics/requirements.txt | 4 ++-- samples/notebooks/requirements-test.txt | 2 +- samples/notebooks/requirements.txt | 4 ++-- samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/samples/desktopapp/requirements-test.txt b/samples/desktopapp/requirements-test.txt index 3bf52c85d..31b836790 100644 --- a/samples/desktopapp/requirements-test.txt +++ b/samples/desktopapp/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-testutils==1.6.4 -pytest==8.4.1 +pytest==8.4.2 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/samples/desktopapp/requirements.txt b/samples/desktopapp/requirements.txt index f86e57e5c..21ccef2fd 100644 --- a/samples/desktopapp/requirements.txt +++ b/samples/desktopapp/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==3.36.0 +google-cloud-bigquery==3.37.0 google-auth-oauthlib==1.2.2 diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index d449b373b..6fb9ba310 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.4.1 +pytest==8.4.2 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index c2bd74bed..c8a93a35e 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,6 +1,6 @@ attrs==25.3.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 click===8.1.8; python_version == '3.9' click==8.2.1; python_version >= '3.10' @@ -13,8 +13,8 @@ geopandas===1.0.1; python_version <= '3.9' geopandas==1.1.1; python_version >= '3.10' google-api-core==2.25.1 google-auth==2.40.3 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-bigquery==3.37.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-core==2.4.3 google-crc32c==1.7.1 google-resumable-media==2.7.2 @@ -29,8 +29,8 @@ proto-plus==1.26.1 pyarrow==21.0.0 pyasn1==0.6.1 pyasn1-modules==0.4.2 -pycparser==2.22 -pyparsing==3.2.3 +pycparser==2.23 +pyparsing==3.2.4 python-dateutil==2.9.0.post0 pytz==2025.2 PyYAML==6.0.2 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index 3bf52c85d..31b836790 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-testutils==1.6.4 -pytest==8.4.1 +pytest==8.4.2 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 7b4f84e8e..d10d53c24 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,6 +1,6 @@ bigquery_magics==0.10.3 db-dtypes==1.4.3 -google.cloud.bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google.cloud.bigquery==3.37.0 +google-cloud-bigquery-storage==2.33.1 ipython===8.18.1 pandas==2.3.2 diff --git a/samples/notebooks/requirements-test.txt b/samples/notebooks/requirements-test.txt index 3bf52c85d..31b836790 100644 --- a/samples/notebooks/requirements-test.txt +++ b/samples/notebooks/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-testutils==1.6.4 -pytest==8.4.1 +pytest==8.4.2 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt index dc22903c7..f65008baa 100644 --- a/samples/notebooks/requirements.txt +++ b/samples/notebooks/requirements.txt @@ -1,7 +1,7 @@ bigquery-magics==0.10.3 db-dtypes==1.4.3 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-bigquery==3.37.0 +google-cloud-bigquery-storage==2.33.1 ipython===8.18.1; python_version == '3.9' ipython==9.5.0; python_version >= '3.10' matplotlib===3.9.2; python_version == '3.9' diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index cef3450e1..901f1df1a 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,5 +1,5 @@ # samples/snippets should be runnable with no "extras" google-cloud-testutils==1.6.4 -pytest==8.4.1 +pytest==8.4.2 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 23da68d60..1fed246f3 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ # samples/snippets should be runnable with no "extras" -google-cloud-bigquery==3.36.0 +google-cloud-bigquery==3.37.0 From 7cad6cf2f95e28b46e529f99b5c4d3cc61603ca4 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 17 Sep 2025 13:23:31 -0700 Subject: [PATCH 3/3] chore(main): release 3.38.0 (#2289) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe721dfde..95db5735c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.38.0](https://github.com/googleapis/python-bigquery/compare/v3.37.0...v3.38.0) (2025-09-15) + + +### Features + +* Add additional query stats ([#2270](https://github.com/googleapis/python-bigquery/issues/2270)) ([7b1b718](https://github.com/googleapis/python-bigquery/commit/7b1b718123afd80c0f68212946e4179bcd6db67f)) + ## [3.37.0](https://github.com/googleapis/python-bigquery/compare/v3.36.0...v3.37.0) (2025-09-08) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index aa24ae04e..22550a8f1 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.37.0" +__version__ = "3.38.0"