From ef2740a158199633b5543a7b6eb19587580792cd Mon Sep 17 00:00:00 2001 From: shollyman Date: Tue, 26 Aug 2025 10:52:30 -0500 Subject: [PATCH 1/7] feat: updates to fastpath query execution (#2268) This PR updates query handling to allow base config properties like job timeout, reservation, and a preview max slots field to leverage the faster path (e.g. using jobs.query rather than jobs.insert). --- google/cloud/bigquery/_job_helpers.py | 3 ++ google/cloud/bigquery/job/base.py | 31 ++++++++++++++++++++ tests/unit/job/test_base.py | 41 +++++++++++++++++++++++++++ tests/unit/job/test_query_config.py | 5 ++++ tests/unit/test__job_helpers.py | 28 ++++++++++++++++++ 5 files changed, 108 insertions(+) diff --git a/google/cloud/bigquery/_job_helpers.py b/google/cloud/bigquery/_job_helpers.py index 6fd561f8c..27e90246f 100644 --- a/google/cloud/bigquery/_job_helpers.py +++ b/google/cloud/bigquery/_job_helpers.py @@ -658,6 +658,9 @@ def _supported_by_jobs_query(request_body: Dict[str, Any]) -> bool: "requestId", "createSession", "writeIncrementalResults", + "jobTimeoutMs", + "reservation", + "maxSlots", } unsupported_keys = request_keys - keys_allowlist diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index f007b9341..1344082be 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -224,6 +224,37 @@ def job_timeout_ms(self, value): else: self._properties.pop("jobTimeoutMs", None) + @property + def max_slots(self) -> Optional[int]: + """The maximum rate of slot consumption to allow for this job. + + If set, the number of slots used to execute the job will be throttled + to try and keep its slot consumption below the requested rate. + This feature is not generally available. + """ + + max_slots = self._properties.get("maxSlots") + if max_slots is not None: + if isinstance(max_slots, str): + return int(max_slots) + if isinstance(max_slots, int): + return max_slots + return None + + @max_slots.setter + def max_slots(self, value): + try: + value = _int_or_none(value) + except ValueError as err: + raise ValueError("Pass an int for max slots, e.g. 100").with_traceback( + err.__traceback__ + ) + + if value is not None: + self._properties["maxSlots"] = str(value) + else: + self._properties.pop("maxSlots", None) + @property def reservation(self): """str: Optional. The reservation that job would use. diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index f5861f645..420904820 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -1276,3 +1276,44 @@ def test_reservation_setter(self): job_config = self._make_one() job_config.reservation = "foo" self.assertEqual(job_config._properties["reservation"], "foo") + + def test_max_slots_miss(self): + job_config = self._make_one() + self.assertEqual(job_config.max_slots, None) + + def test_max_slots_set_and_clear(self): + job_config = self._make_one() + job_config.max_slots = 14 + self.assertEqual(job_config.max_slots, 14) + job_config.max_slots = None + self.assertEqual(job_config.max_slots, None) + + def test_max_slots_hit_str(self): + job_config = self._make_one() + job_config._properties["maxSlots"] = "4" + self.assertEqual(job_config.max_slots, 4) + + def test_max_slots_hit_int(self): + job_config = self._make_one() + job_config._properties["maxSlots"] = int(3) + self.assertEqual(job_config.max_slots, 3) + + def test_max_slots_hit_invalid(self): + job_config = self._make_one() + job_config._properties["maxSlots"] = object() + self.assertEqual(job_config.max_slots, None) + + def test_max_slots_update_in_place(self): + job_config = self._make_one() + job_config.max_slots = 45 # update in place + self.assertEqual(job_config.max_slots, 45) + + def test_max_slots_setter_invalid(self): + job_config = self._make_one() + with self.assertRaises(ValueError): + job_config.max_slots = "foo" + + def test_max_slots_setter(self): + job_config = self._make_one() + job_config.max_slots = 123 + self.assertEqual(job_config._properties["maxSlots"], "123") diff --git a/tests/unit/job/test_query_config.py b/tests/unit/job/test_query_config.py index e0878d067..a63a14b73 100644 --- a/tests/unit/job/test_query_config.py +++ b/tests/unit/job/test_query_config.py @@ -172,6 +172,11 @@ def test_incremental_results(self): config.write_incremental_results = True self.assertEqual(config.write_incremental_results, True) + def test_max_slots(self): + config = self._get_target_class()() + config.max_slots = 99 + self.assertEqual(config.max_slots, 99) + def test_create_session(self): config = self._get_target_class()() self.assertIsNone(config.create_session) diff --git a/tests/unit/test__job_helpers.py b/tests/unit/test__job_helpers.py index 1f543f033..10cbefe13 100644 --- a/tests/unit/test__job_helpers.py +++ b/tests/unit/test__job_helpers.py @@ -200,6 +200,19 @@ def make_query_response( make_query_request({"writeIncrementalResults": True}), id="job_config-with-incremental-results", ), + pytest.param( + job_query.QueryJobConfig( + reservation="foo", + max_slots=100, + ), + make_query_request( + { + "maxSlots": "100", + "reservation": "foo", + } + ), + id="job_config-with-reservation-and-slots", + ), ), ) def test__to_query_request(job_config, expected): @@ -1048,6 +1061,21 @@ def test_make_job_id_w_job_id_overrides_prefix(): True, id="write_incremental_results", ), + pytest.param( + job_query.QueryJobConfig(job_timeout_ms=1000), + True, + id="job_timeout_ms", + ), + pytest.param( + job_query.QueryJobConfig(reservation="foo"), + True, + id="reservation", + ), + pytest.param( + job_query.QueryJobConfig(max_slots=20), + True, + id="max_slots", + ), ), ) def test_supported_by_jobs_query_from_queryjobconfig( From 43527af24e56994357205b482a86b805950d2d0f Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 3 Sep 2025 18:27:17 +0200 Subject: [PATCH 2/7] chore(deps): update all dependencies (#2259) --- samples/desktopapp/requirements.txt | 2 +- samples/geography/requirements.txt | 10 +++++----- samples/magics/requirements.txt | 6 +++--- samples/notebooks/requirements.txt | 10 +++++----- samples/snippets/requirements.txt | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/samples/desktopapp/requirements.txt b/samples/desktopapp/requirements.txt index e7a02eca5..f86e57e5c 100644 --- a/samples/desktopapp/requirements.txt +++ b/samples/desktopapp/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==3.35.1 +google-cloud-bigquery==3.36.0 google-auth-oauthlib==1.2.2 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index fa54cc229..c2bd74bed 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,7 +1,7 @@ attrs==25.3.0 certifi==2025.8.3 cffi==1.17.1 -charset-normalizer==3.4.2 +charset-normalizer==3.4.3 click===8.1.8; python_version == '3.9' click==8.2.1; python_version >= '3.10' click-plugins==1.1.1.2 @@ -13,7 +13,7 @@ geopandas===1.0.1; python_version <= '3.9' geopandas==1.1.1; python_version >= '3.10' google-api-core==2.25.1 google-auth==2.40.3 -google-cloud-bigquery==3.35.1 +google-cloud-bigquery==3.36.0 google-cloud-bigquery-storage==2.32.0 google-cloud-core==2.4.3 google-crc32c==1.7.1 @@ -24,7 +24,7 @@ idna==3.10 munch==4.0.0 mypy-extensions==1.1.0 packaging==25.0 -pandas==2.3.1 +pandas==2.3.2 proto-plus==1.26.1 pyarrow==21.0.0 pyasn1==0.6.1 @@ -34,11 +34,11 @@ pyparsing==3.2.3 python-dateutil==2.9.0.post0 pytz==2025.2 PyYAML==6.0.2 -requests==2.32.4 +requests==2.32.5 rsa==4.9.1 Shapely===2.0.7; python_version == '3.9' Shapely==2.1.1; python_version >= '3.10' six==1.17.0 -typing-extensions==4.14.1 +typing-extensions==4.15.0 typing-inspect==0.9.0 urllib3==2.5.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index e7230053c..7b4f84e8e 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,6 +1,6 @@ -bigquery_magics==0.10.2 +bigquery_magics==0.10.3 db-dtypes==1.4.3 -google.cloud.bigquery==3.35.1 +google.cloud.bigquery==3.36.0 google-cloud-bigquery-storage==2.32.0 ipython===8.18.1 -pandas==2.3.1 +pandas==2.3.2 diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt index 829f08f47..dc22903c7 100644 --- a/samples/notebooks/requirements.txt +++ b/samples/notebooks/requirements.txt @@ -1,9 +1,9 @@ -bigquery-magics==0.10.2 +bigquery-magics==0.10.3 db-dtypes==1.4.3 -google-cloud-bigquery==3.35.1 +google-cloud-bigquery==3.36.0 google-cloud-bigquery-storage==2.32.0 ipython===8.18.1; python_version == '3.9' -ipython==9.4.0; python_version >= '3.10' +ipython==9.5.0; python_version >= '3.10' matplotlib===3.9.2; python_version == '3.9' -matplotlib==3.10.5; python_version >= '3.10' -pandas==2.3.1 +matplotlib==3.10.6; python_version >= '3.10' +pandas==2.3.2 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index afa62b6b8..23da68d60 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ # samples/snippets should be runnable with no "extras" -google-cloud-bigquery==3.35.1 +google-cloud-bigquery==3.36.0 From 435ecdb62d8402fea317763e48934fa510ce8026 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 3 Sep 2025 14:30:49 -0400 Subject: [PATCH 3/7] bug: updates `__eq__` comparison on TableConstraint (#2274) * bug: updates __eq__ comparison on TableConstraint * updates tests * moves test out of class to accommodate pytest params --- google/cloud/bigquery/table.py | 6 +- tests/unit/test_table.py | 144 ++++++++++++++++----------------- 2 files changed, 71 insertions(+), 79 deletions(-) diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 219b31467..5efcb1958 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -3574,9 +3574,9 @@ def __init__( def __eq__(self, other): if not isinstance(other, TableConstraints) and other is not None: raise TypeError("The value provided is not a BigQuery TableConstraints.") - return ( - self.primary_key == other.primary_key if other.primary_key else None - ) and (self.foreign_keys == other.foreign_keys if other.foreign_keys else None) + return self.primary_key == ( + other.primary_key if other.primary_key else None + ) and self.foreign_keys == (other.foreign_keys if other.foreign_keys else None) @classmethod def from_api_repr(cls, resource: Dict[str, Any]) -> "TableConstraints": diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index eb2c8d9ec..af31d116b 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -6322,82 +6322,6 @@ def test_constructor_explicit_with_none(self): self.assertIsNone(table_constraint.primary_key) self.assertIsNone(table_constraint.foreign_keys) - def test__eq__primary_key_mismatch(self): - from google.cloud.bigquery.table import ( - PrimaryKey, - ForeignKey, - TableReference, - ColumnReference, - ) - - foriegn_keys = [ - ForeignKey( - name="my_fk_id", - referenced_table=TableReference.from_string( - "my-project.my-dataset.my-table" - ), - column_references=[ - ColumnReference(referencing_column="id", referenced_column="id"), - ], - ), - ] - - table_constraint = self._make_one( - primary_key=PrimaryKey(columns=["my_pk_id"]), - foreign_keys=foriegn_keys, - ) - other_table_constraint = self._make_one( - primary_key=PrimaryKey(columns=["my_other_pk_id"]), - foreign_keys=foriegn_keys, - ) - - self.assertNotEqual(table_constraint, other_table_constraint) - - def test__eq__foreign_keys_mismatch(self): - from google.cloud.bigquery.table import ( - PrimaryKey, - ForeignKey, - TableReference, - ColumnReference, - ) - - primary_key = PrimaryKey(columns=["my_pk_id"]) - - table_constraint = self._make_one( - primary_key=primary_key, - foreign_keys=[ - ForeignKey( - name="my_fk_id", - referenced_table=TableReference.from_string( - "my-project.my-dataset.my-table" - ), - column_references=[ - ColumnReference( - referencing_column="id", referenced_column="id" - ), - ], - ), - ], - ) - other_table_constraint = self._make_one( - primary_key=primary_key, - foreign_keys=[ - ForeignKey( - name="my_other_fk_id", - referenced_table=TableReference.from_string( - "my-project.my-dataset.my-other-table" - ), - column_references=[ - ColumnReference( - referencing_column="other_id", referenced_column="other_id" - ), - ], - ), - ], - ) - - self.assertNotEqual(table_constraint, other_table_constraint) - def test__eq__other_type(self): from google.cloud.bigquery.table import ( PrimaryKey, @@ -6615,6 +6539,74 @@ def test_to_api_repr_empty_constraints(self): self.assertEqual(instance.to_api_repr(), expected) +@pytest.mark.parametrize( + "self_pk_name,self_fk_name,other_pk_name,other_fk_name,expected_equal", + [ + (None, None, None, None, True), + ("pkey", None, "pkey", None, True), + ("pkey", "fkey", "pkey", "fkey", True), + (None, "fkey", None, "fkey", True), + ("pkey", None, "pkey_no_match", None, False), + ("pkey", "fkey", "pkey_no_match", "fkey_no_match", False), + (None, "fkey", None, "fkey_no_match", False), + ("pkey", "fkey", "pkey_no_match", "fkey", False), + ("pkey", "fkey", "pkey", "fkey_no_match", False), + ], +) +def test_table_constraint_eq_parametrized( + self_pk_name, self_fk_name, other_pk_name, other_fk_name, expected_equal +): + # Imports are placed here to ensure they are self-contained for this example. + # In a real test file, they would likely be at the top of the file. + from google.cloud.bigquery.table import ( + ColumnReference, + ForeignKey, + PrimaryKey, + TableReference, + TableConstraints, + ) + + # Helper function to create a PrimaryKey object or None + def _create_primary_key(name): + if name is None: + return None + return PrimaryKey(columns=[name]) + + # Helper function to create a list of ForeignKey objects or None + def _create_foreign_keys(name): + if name is None: + return None + # Using a generic referenced_table and column_references for simplicity + # The 'name' parameter ensures different ForeignKey objects for different names + return [ + ForeignKey( + name=name, + referenced_table=TableReference.from_string( + f"my-project.my-dataset.{name}_referenced_table" + ), + column_references=[ + ColumnReference( + referencing_column=f"{name}_ref_col", + referenced_column=f"{name}_pk_col", + ) + ], + ) + ] + + # Create the two TableConstraints instances for comparison + tc1 = TableConstraints( + primary_key=_create_primary_key(self_pk_name), + foreign_keys=_create_foreign_keys(self_fk_name), + ) + tc2 = TableConstraints( + primary_key=_create_primary_key(other_pk_name), + foreign_keys=_create_foreign_keys(other_fk_name), + ) + + # Assert the equality based on the expected outcome + assert (tc1 == tc2) == expected_equal + + class TestExternalCatalogTableOptions: PROJECT = "test-project" DATASET_ID = "test_dataset" From 8a13c12905ffcb3dbb6086a61df37556f0c2cd31 Mon Sep 17 00:00:00 2001 From: shollyman Date: Thu, 4 Sep 2025 13:25:49 -0500 Subject: [PATCH 4/7] docs: clarify the api_method arg for client.query() (#2277) * docs: clarify the api_method arg for client.query() --- google/cloud/bigquery/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 4ca2cb428..ea592852a 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -3519,7 +3519,8 @@ def query( specified here becomes the default ``job_retry`` for ``result()``, where it can also be specified. api_method (Union[str, enums.QueryApiMethod]): - Method with which to start the query job. + Method with which to start the query job. By default, + the jobs.insert API is used for starting a query. See :class:`google.cloud.bigquery.enums.QueryApiMethod` for details on the difference between the query start methods. From 33ea29616c06a2e2a106a785d216e784737ae386 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Sat, 6 Sep 2025 10:16:44 -0700 Subject: [PATCH 5/7] fix: remove deepcopy while setting properties for _QueryResults (#2280) --- google/cloud/bigquery/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 7f70f6a2a..170ed2976 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -1400,7 +1400,7 @@ def _set_properties(self, api_response): api_response (Dict): Response returned from an API call """ self._properties.clear() - self._properties.update(copy.deepcopy(api_response)) + self._properties.update(api_response) def _query_param_from_api_repr(resource): From 6e88d7dbe42ebfc35986da665d656b49ac481db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Mon, 8 Sep 2025 09:58:27 -0500 Subject: [PATCH 6/7] docs: clarify that the presence of `XyzJob.errors` doesn't necessarily mean that the job has not completed or was unsuccessful (#2278) Internal issue b/440349994 --- google/cloud/bigquery/job/base.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 1344082be..9b7ddb82d 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -693,7 +693,12 @@ def transaction_info(self) -> Optional[TransactionInfo]: @property def error_result(self): - """Error information about the job as a whole. + """Output only. Final error result of the job. + + If present, indicates that the job has completed and was unsuccessful. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobStatus.FIELDS.error_result Returns: Optional[Mapping]: the error information (None until set from the server). @@ -704,7 +709,13 @@ def error_result(self): @property def errors(self): - """Information about individual errors generated by the job. + """Output only. The first errors encountered during the running of the job. + + The final message includes the number of errors that caused the process to stop. + Errors here do not necessarily mean that the job has not completed or was unsuccessful. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobStatus.FIELDS.errors Returns: Optional[List[Mapping]]: @@ -716,7 +727,12 @@ def errors(self): @property def state(self): - """Status of the job. + """Output only. Running state of the job. + + Valid states include 'PENDING', 'RUNNING', and 'DONE'. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobStatus.FIELDS.state Returns: Optional[str]: From 4b0ef0cfcf7def138e43a22223abfcbefc330da2 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:01:33 -0700 Subject: [PATCH 7/7] chore(main): release 3.37.0 (#2269) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 18 ++++++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62352c344..fe721dfde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.37.0](https://github.com/googleapis/python-bigquery/compare/v3.36.0...v3.37.0) (2025-09-08) + + +### Features + +* Updates to fastpath query execution ([#2268](https://github.com/googleapis/python-bigquery/issues/2268)) ([ef2740a](https://github.com/googleapis/python-bigquery/commit/ef2740a158199633b5543a7b6eb19587580792cd)) + + +### Bug Fixes + +* Remove deepcopy while setting properties for _QueryResults ([#2280](https://github.com/googleapis/python-bigquery/issues/2280)) ([33ea296](https://github.com/googleapis/python-bigquery/commit/33ea29616c06a2e2a106a785d216e784737ae386)) + + +### Documentation + +* Clarify that the presence of `XyzJob.errors` doesn't necessarily mean that the job has not completed or was unsuccessful ([#2278](https://github.com/googleapis/python-bigquery/issues/2278)) ([6e88d7d](https://github.com/googleapis/python-bigquery/commit/6e88d7dbe42ebfc35986da665d656b49ac481db4)) +* Clarify the api_method arg for client.query() ([#2277](https://github.com/googleapis/python-bigquery/issues/2277)) ([8a13c12](https://github.com/googleapis/python-bigquery/commit/8a13c12905ffcb3dbb6086a61df37556f0c2cd31)) + ## [3.36.0](https://github.com/googleapis/python-bigquery/compare/v3.35.1...v3.36.0) (2025-08-20) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index a8f4c8e14..aa24ae04e 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.36.0" +__version__ = "3.37.0"