Skip to content

Commit 54d3dc6

Browse files
chalmerlowechalmerlowe
authored
feat: adds datetime_format as an option (#2236)
* feat: adds datetime_format as an option * updates docstrings
1 parent 371ad29 commit 54d3dc6

File tree

5 files changed

+65
-0
lines changed

5 files changed

+65
-0
lines changed

‎google/cloud/bigquery/external_config.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,21 @@ def date_format(self) -> Optional[str]:
862862
def date_format(self, value: Optional[str]):
863863
self._properties["dateFormat"] = value
864864

865+
@property
866+
def datetime_format(self) -> Optional[str]:
867+
"""Optional[str]: Format used to parse DATETIME values. Supports C-style
868+
and SQL-style values.
869+
870+
See:
871+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.datetime_format
872+
"""
873+
result = self._properties.get("datetimeFormat")
874+
return typing.cast(str, result)
875+
876+
@datetime_format.setter
877+
def datetime_format(self, value: Optional[str]):
878+
self._properties["datetimeFormat"] = value
879+
865880
@property
866881
def time_zone(self) -> Optional[str]:
867882
"""Optional[str]: Time zone used when parsing timestamp values that do not

‎google/cloud/bigquery/job/load.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,19 @@ def date_format(self) -> Optional[str]:
561561
def date_format(self, value: Optional[str]):
562562
self._set_sub_prop("dateFormat", value)
563563

564+
@property
565+
def datetime_format(self) -> Optional[str]:
566+
"""Optional[str]: Date format used for parsing DATETIME values.
567+
568+
See:
569+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.datetime_format
570+
"""
571+
return self._get_sub_prop("datetimeFormat")
572+
573+
@datetime_format.setter
574+
def datetime_format(self, value: Optional[str]):
575+
self._set_sub_prop("datetimeFormat", value)
576+
564577
@property
565578
def time_zone(self) -> Optional[str]:
566579
"""Optional[str]: Default time zone that will apply when parsing timestamp
@@ -949,6 +962,13 @@ def date_format(self):
949962
"""
950963
return self.configuration.date_format
951964

965+
@property
966+
def datetime_format(self):
967+
"""See
968+
:attr:`google.cloud.bigquery.job.LoadJobConfig.datetime_format`.
969+
"""
970+
return self.configuration.datetime_format
971+
952972
@property
953973
def time_zone(self):
954974
"""See

‎tests/unit/job/test_load.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def _setUpConstants(self):
3838
self.OUTPUT_ROWS = 345
3939
self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
4040
self.DATE_FORMAT = "%Y-%m-%d"
41+
self.DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
4142
self.TIME_ZONE = "UTC"
4243
self.TIME_FORMAT = "%H:%M:%S"
4344
self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
@@ -47,6 +48,7 @@ def _make_resource(self, started=False, ended=False):
4748
config = resource["configuration"]["load"]
4849
config["sourceUris"] = [self.SOURCE1]
4950
config["dateFormat"] = self.DATE_FORMAT
51+
config["datetimeFormat"] = self.DATETIME_FORMAT
5052
config["timeZone"] = self.TIME_ZONE
5153
config["timeFormat"] = self.TIME_FORMAT
5254
config["timestampFormat"] = self.TIMESTAMP_FORMAT
@@ -164,6 +166,10 @@ def _verifyResourceProperties(self, job, resource):
164166
self.assertEqual(job.date_format, config["dateFormat"])
165167
else:
166168
self.assertIsNone(job.date_format)
169+
if "datetimeFormat" in config:
170+
self.assertEqual(job.datetime_format, config["datetimeFormat"])
171+
else:
172+
self.assertIsNone(job.datetime_format)
167173
if "timeZone" in config:
168174
self.assertEqual(job.time_zone, config["timeZone"])
169175
else:
@@ -219,6 +225,7 @@ def test_ctor(self):
219225
self.assertIsNone(job.schema_update_options)
220226
self.assertIsNone(job.reference_file_schema_uri)
221227
self.assertIsNone(job.date_format)
228+
self.assertIsNone(job.datetime_format)
222229
self.assertIsNone(job.time_zone)
223230
self.assertIsNone(job.time_format)
224231
self.assertIsNone(job.timestamp_format)
@@ -618,6 +625,7 @@ def test_begin_w_alternate_client(self):
618625
},
619626
"schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
620627
"dateFormat": self.DATE_FORMAT,
628+
"datetimeFormat": self.DATETIME_FORMAT,
621629
"timeZone": self.TIME_ZONE,
622630
"timeFormat": self.TIME_FORMAT,
623631
"timestampFormat": self.TIMESTAMP_FORMAT,
@@ -651,6 +659,7 @@ def test_begin_w_alternate_client(self):
651659
config.schema_update_options = [SchemaUpdateOption.ALLOW_FIELD_ADDITION]
652660
config.reference_file_schema_uri = "gs://path/to/reference"
653661
config.date_format = self.DATE_FORMAT
662+
config.datetime_format = self.DATETIME_FORMAT
654663
config.time_zone = self.TIME_ZONE
655664
config.time_format = self.TIME_FORMAT
656665
config.timestamp_format = self.TIMESTAMP_FORMAT

‎tests/unit/job/test_load_config.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -844,6 +844,22 @@ def test_date_format_setter(self):
844844
config.date_format = date_format
845845
self.assertEqual(config._properties["load"]["dateFormat"], date_format)
846846

847+
def test_datetime_format_missing(self):
848+
config = self._get_target_class()()
849+
self.assertIsNone(config.datetime_format)
850+
851+
def test_datetime_format_hit(self):
852+
datetime_format = "%Y-%m-%dT%H:%M:%S"
853+
config = self._get_target_class()()
854+
config._properties["load"]["datetimeFormat"] = datetime_format
855+
self.assertEqual(config.datetime_format, datetime_format)
856+
857+
def test_datetime_format_setter(self):
858+
datetime_format = "YYYY/MM/DD HH24:MI:SS"
859+
config = self._get_target_class()()
860+
config.datetime_format = datetime_format
861+
self.assertEqual(config._properties["load"]["datetimeFormat"], datetime_format)
862+
847863
def test_time_zone_missing(self):
848864
config = self._get_target_class()()
849865
self.assertIsNone(config.time_zone)

‎tests/unit/test_external_config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
class TestExternalConfig(unittest.TestCase):
2727
SOURCE_URIS = ["gs://foo", "gs://bar"]
2828
DATE_FORMAT = "MM/DD/YYYY"
29+
DATETIME_FORMAT = "MM/DD/YYYY HH24:MI:SS"
2930
TIME_ZONE = "America/Los_Angeles"
3031
TIME_FORMAT = "HH24:MI:SS"
3132
TIMESTAMP_FORMAT = "MM/DD/YYYY HH24:MI:SS.FF6 TZR"
@@ -38,6 +39,7 @@ class TestExternalConfig(unittest.TestCase):
3839
"ignoreUnknownValues": False,
3940
"compression": "compression",
4041
"dateFormat": DATE_FORMAT,
42+
"datetimeFormat": DATETIME_FORMAT,
4143
"timeZone": TIME_ZONE,
4244
"timeFormat": TIME_FORMAT,
4345
"timestampFormat": TIMESTAMP_FORMAT,
@@ -88,6 +90,7 @@ def test_to_api_repr_base(self):
8890
ec.schema = [schema.SchemaField("full_name", "STRING", mode="REQUIRED")]
8991

9092
ec.date_format = self.DATE_FORMAT
93+
ec.datetime_format = self.DATETIME_FORMAT
9194
ec.time_zone = self.TIME_ZONE
9295
ec.time_format = self.TIME_FORMAT
9396
ec.timestamp_format = self.TIMESTAMP_FORMAT
@@ -106,6 +109,7 @@ def test_to_api_repr_base(self):
106109
"connectionId": "path/to/connection",
107110
"schema": exp_schema,
108111
"dateFormat": self.DATE_FORMAT,
112+
"datetimeFormat": self.DATETIME_FORMAT,
109113
"timeZone": self.TIME_ZONE,
110114
"timeFormat": self.TIME_FORMAT,
111115
"timestampFormat": self.TIMESTAMP_FORMAT,
@@ -145,6 +149,7 @@ def _verify_base(self, ec):
145149
self.assertEqual(ec.max_bad_records, 17)
146150
self.assertEqual(ec.source_uris, self.SOURCE_URIS)
147151
self.assertEqual(ec.date_format, self.DATE_FORMAT)
152+
self.assertEqual(ec.datetime_format, self.DATETIME_FORMAT)
148153
self.assertEqual(ec.time_zone, self.TIME_ZONE)
149154
self.assertEqual(ec.time_format, self.TIME_FORMAT)
150155
self.assertEqual(ec.timestamp_format, self.TIMESTAMP_FORMAT)

0 commit comments

Comments
 (0)