diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ee94722ab..108063d4d 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc + digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 6572e5982..01affbae5 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,9 +1,12 @@ # https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings +# Allow merge commits to sync main and v3 with fewer conflicts. +mergeCommitAllowed: true # Rules for main branch protection branchProtectionRules: # Identifies the protection rule pattern. Name of the branch to be protected. # Defaults to `main` - pattern: main + requiresLinearHistory: true requiresCodeOwnerReviews: true requiresStrictStatusChecks: true requiredStatusCheckContexts: @@ -15,6 +18,7 @@ branchProtectionRules: - 'Samples - Python 3.7' - 'Samples - Python 3.8' - pattern: v3 + requiresLinearHistory: false requiresCodeOwnerReviews: true requiresStrictStatusChecks: true requiredStatusCheckContexts: diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 0c99ae611..41b86fc29 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -30,6 +30,7 @@ env_vars: { env_vars: { key: "V2_STAGING_BUCKET" + # Push google cloud library docs to the Cloud RAD bucket `docs-staging-v2` value: "docs-staging-v2" } diff --git a/.kokoro/samples/python3.10/common.cfg b/.kokoro/samples/python3.10/common.cfg new file mode 100644 index 000000000..da4003d76 --- /dev/null +++ b/.kokoro/samples/python3.10/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.10" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-310" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.10/continuous.cfg b/.kokoro/samples/python3.10/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.10/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.10/periodic-head.cfg b/.kokoro/samples/python3.10/periodic-head.cfg new file mode 100644 index 000000000..5aa01bab5 --- /dev/null +++ b/.kokoro/samples/python3.10/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.10/periodic.cfg b/.kokoro/samples/python3.10/periodic.cfg new file mode 100644 index 000000000..71cd1e597 --- /dev/null +++ b/.kokoro/samples/python3.10/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.10/presubmit.cfg b/.kokoro/samples/python3.10/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.10/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d15f22851..0d45d501d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.29.0](https://www.github.com/googleapis/python-bigquery/compare/v2.28.1...v2.29.0) (2021-10-27) + + +### Features + +* add `QueryJob.schema` property for dry run queries ([#1014](https://www.github.com/googleapis/python-bigquery/issues/1014)) ([2937fa1](https://www.github.com/googleapis/python-bigquery/commit/2937fa1386898766c561579fd39d42958182d260)) +* add session and connection properties to QueryJobConfig ([#1024](https://www.github.com/googleapis/python-bigquery/issues/1024)) ([e4c94f4](https://www.github.com/googleapis/python-bigquery/commit/e4c94f446c27eb474f30b033c1b62d11bd0acd98)) +* add support for INTERVAL data type to `list_rows` ([#840](https://www.github.com/googleapis/python-bigquery/issues/840)) ([e37380a](https://www.github.com/googleapis/python-bigquery/commit/e37380a959cbd5bb9cbbf6807f0a8ea147e0a713)) +* allow queryJob.result() to be called on a dryRun ([#1015](https://www.github.com/googleapis/python-bigquery/issues/1015)) ([685f06a](https://www.github.com/googleapis/python-bigquery/commit/685f06a5e7b5df17a53e9eb340ff04ecd1e51d1d)) + + +### Documentation + +* document ScriptStatistics and other missing resource classes ([#1023](https://www.github.com/googleapis/python-bigquery/issues/1023)) ([6679109](https://www.github.com/googleapis/python-bigquery/commit/66791093c61f262ea063d2a7950fc643915ee693)) +* fix formatting of generated client docstrings ([#1009](https://www.github.com/googleapis/python-bigquery/issues/1009)) ([f7b0ee4](https://www.github.com/googleapis/python-bigquery/commit/f7b0ee45a664295ccc9f209eeeac122af8de3c80)) + + +### Dependencies + +* allow pyarrow 6.x ([#1031](https://www.github.com/googleapis/python-bigquery/issues/1031)) ([1c2de74](https://www.github.com/googleapis/python-bigquery/commit/1c2de74a55046a343bcf9474f67100a82fb05401)) + ### [2.28.1](https://www.github.com/googleapis/python-bigquery/compare/v2.28.0...v2.28.1) (2021-10-07) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8aecf9dd2..f183b63b4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.6, 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.9 -- -k + $ nox -s unit-3.10 -- -k .. note:: @@ -225,11 +225,13 @@ We support: - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ +- `Python 3.10`_ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ +.. _Python 3.10: https://docs.python.org/3.10/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/docs/conf.py b/docs/conf.py index 329951636..0784da0b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -366,8 +366,9 @@ "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - "pandas": ("http://pandas.pydata.org/pandas-docs/stable/", None), + "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None), "geopandas": ("https://geopandas.org/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/dev", None), } diff --git a/docs/job_base.rst b/docs/job_base.rst new file mode 100644 index 000000000..f5ef06b88 --- /dev/null +++ b/docs/job_base.rst @@ -0,0 +1,5 @@ +Common Job Resource Classes +=========================== + +.. automodule:: google.cloud.bigquery.job.base + :members: diff --git a/docs/query.rst b/docs/query.rst new file mode 100644 index 000000000..d3cb8fe83 --- /dev/null +++ b/docs/query.rst @@ -0,0 +1,5 @@ +Query Resource Classes +====================== + +.. automodule:: google.cloud.bigquery.query + :members: diff --git a/docs/reference.rst b/docs/reference.rst index d2d2eed31..00f64746f 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -47,7 +47,6 @@ Job Classes job.CopyJob job.LoadJob job.ExtractJob - job.UnknownJob Job-Related Types ----------------- @@ -68,7 +67,11 @@ Job-Related Types job.SourceFormat job.WriteDisposition job.SchemaUpdateOption - job.TransactionInfo + +.. toctree:: + :maxdepth: 2 + + job_base Dataset @@ -134,14 +137,10 @@ Schema Query ===== -.. autosummary:: - :toctree: generated +.. toctree:: + :maxdepth: 2 - query.ArrayQueryParameter - query.ScalarQueryParameter - query.ScalarQueryParameterType - query.StructQueryParameter - query.UDFResource + query Retries diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index d2b1dd26d..b3c492125 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -52,6 +52,7 @@ from google.cloud.bigquery.external_config import ExternalSourceFormat from google.cloud.bigquery.format_options import AvroOptions from google.cloud.bigquery.format_options import ParquetOptions +from google.cloud.bigquery.job.base import SessionInfo from google.cloud.bigquery.job import Compression from google.cloud.bigquery.job import CopyJob from google.cloud.bigquery.job import CopyJobConfig @@ -77,6 +78,7 @@ from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.query import ArrayQueryParameter from google.cloud.bigquery.query import ArrayQueryParameterType +from google.cloud.bigquery.query import ConnectionProperty from google.cloud.bigquery.query import ScalarQueryParameter from google.cloud.bigquery.query import ScalarQueryParameterType from google.cloud.bigquery.query import StructQueryParameter @@ -104,6 +106,7 @@ "__version__", "Client", # Queries + "ConnectionProperty", "QueryJob", "QueryJobConfig", "ArrayQueryParameter", @@ -132,6 +135,7 @@ "ExtractJobConfig", "LoadJob", "LoadJobConfig", + "SessionInfo", "UnknownJob", # Models "Model", diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index d7189d322..e95d38545 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -19,8 +19,9 @@ import decimal import math import re -from typing import Any, Union +from typing import Any, Optional, Union +from dateutil import relativedelta from google.cloud._helpers import UTC from google.cloud._helpers import _date_from_iso8601_date from google.cloud._helpers import _datetime_from_microseconds @@ -45,6 +46,14 @@ re.VERBOSE, ) +# BigQuery sends INTERVAL data in "canonical format" +# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#interval_type +_INTERVAL_PATTERN = re.compile( + r"(?P-?)(?P\d+)-(?P\d+) " + r"(?P-?\d+) " + r"(?P-?)(?P\d+):(?P\d+):(?P\d+)\.?(?P\d*)?$" +) + _MIN_PYARROW_VERSION = packaging.version.Version("3.0.0") _MIN_BQ_STORAGE_VERSION = packaging.version.Version("2.0.0") _BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION = packaging.version.Version("2.6.0") @@ -191,6 +200,41 @@ def _int_from_json(value, field): return int(value) +def _interval_from_json( + value: Optional[str], field +) -> Optional[relativedelta.relativedelta]: + """Coerce 'value' to an interval, if set or not nullable.""" + if not _not_null(value, field): + return None + if value is None: + raise TypeError(f"got {value} for REQUIRED field: {repr(field)}") + + parsed = _INTERVAL_PATTERN.match(value) + if parsed is None: + raise ValueError(f"got interval: '{value}' with unexpected format") + + calendar_sign = -1 if parsed.group("calendar_sign") == "-" else 1 + years = calendar_sign * int(parsed.group("years")) + months = calendar_sign * int(parsed.group("months")) + days = int(parsed.group("days")) + time_sign = -1 if parsed.group("time_sign") == "-" else 1 + hours = time_sign * int(parsed.group("hours")) + minutes = time_sign * int(parsed.group("minutes")) + seconds = time_sign * int(parsed.group("seconds")) + fraction = parsed.group("fraction") + microseconds = time_sign * int(fraction.ljust(6, "0")[:6]) if fraction else 0 + + return relativedelta.relativedelta( + years=years, + months=months, + days=days, + hours=hours, + minutes=minutes, + seconds=seconds, + microseconds=microseconds, + ) + + def _float_from_json(value, field): """Coerce 'value' to a float, if set or not nullable.""" if _not_null(value, field): @@ -327,6 +371,7 @@ def _record_from_json(value, field): _CELLDATA_FROM_JSON = { "INTEGER": _int_from_json, "INT64": _int_from_json, + "INTERVAL": _interval_from_json, "FLOAT": _float_from_json, "FLOAT64": _float_from_json, "NUMERIC": _decimal_from_json, diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index a8a1c1e16..9cb6af8f0 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -549,7 +549,7 @@ def _dataset_from_arg(self, dataset): def create_dataset( self, - dataset: Union[str, Dataset, DatasetReference], + dataset: Union[str, Dataset, DatasetReference, DatasetListItem], exists_ok: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, @@ -679,7 +679,7 @@ def create_routine( def create_table( self, - table: Union[str, Table, TableReference], + table: Union[str, Table, TableReference, TableListItem], exists_ok: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, @@ -693,6 +693,7 @@ def create_table( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): A :class:`~google.cloud.bigquery.table.Table` to create. @@ -1295,7 +1296,7 @@ def update_table( def list_models( self, - dataset: Union[Dataset, DatasetReference, str], + dataset: Union[Dataset, DatasetReference, DatasetListItem, str], max_results: int = None, page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, @@ -1372,7 +1373,7 @@ def api_request(*args, **kwargs): def list_routines( self, - dataset: Union[Dataset, DatasetReference, str], + dataset: Union[Dataset, DatasetReference, DatasetListItem, str], max_results: int = None, page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, @@ -1449,7 +1450,7 @@ def api_request(*args, **kwargs): def list_tables( self, - dataset: Union[Dataset, DatasetReference, str], + dataset: Union[Dataset, DatasetReference, DatasetListItem, str], max_results: int = None, page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, @@ -1525,7 +1526,7 @@ def api_request(*args, **kwargs): def delete_dataset( self, - dataset: Union[Dataset, DatasetReference, str], + dataset: Union[Dataset, DatasetReference, DatasetListItem, str], delete_contents: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index d67cebd4c..0eaaffd2e 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -254,6 +254,7 @@ class SqlTypeNames(str, enum.Enum): DATE = "DATE" TIME = "TIME" DATETIME = "DATETIME" + INTERVAL = "INTERVAL" # NOTE: not available in legacy types class SqlParameterScalarTypes: diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 698181092..88d6bec14 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -19,7 +19,7 @@ import http import threading import typing -from typing import Dict, Optional +from typing import Dict, Optional, Sequence from google.api_core import exceptions import google.api_core.future.polling @@ -193,7 +193,8 @@ def parent_job_id(self): return _helpers._get_sub_prop(self._properties, ["statistics", "parentJobId"]) @property - def script_statistics(self): + def script_statistics(self) -> Optional["ScriptStatistics"]: + """Statistics for a child job of a script.""" resource = _helpers._get_sub_prop( self._properties, ["statistics", "scriptStatistics"] ) @@ -201,6 +202,19 @@ def script_statistics(self): return None return ScriptStatistics(resource) + @property + def session_info(self) -> Optional["SessionInfo"]: + """[Preview] Information of the session if this job is part of one. + + .. versionadded:: 2.29.0 + """ + resource = _helpers._get_sub_prop( + self._properties, ["statistics", "sessionInfo"] + ) + if resource is None: + return None + return SessionInfo(resource) + @property def num_child_jobs(self): """The number of child jobs executed. @@ -968,9 +982,8 @@ def __init__(self, resource): self._properties = resource @property - def stack_frames(self): - """List[ScriptStackFrame]: Stack trace where the current evaluation - happened. + def stack_frames(self) -> Sequence[ScriptStackFrame]: + """Stack trace where the current evaluation happened. Shows line/column/procedure name of each frame on the stack at the point where the current evaluation happened. @@ -982,7 +995,7 @@ def stack_frames(self): ] @property - def evaluation_kind(self): + def evaluation_kind(self) -> Optional[str]: """str: Indicates the type of child job. Possible values include ``STATEMENT`` and ``EXPRESSION``. @@ -990,6 +1003,24 @@ def evaluation_kind(self): return self._properties.get("evaluationKind") +class SessionInfo: + """[Preview] Information of the session if this job is part of one. + + .. versionadded:: 2.29.0 + + Args: + resource (Map[str, Any]): JSON representation of object. + """ + + def __init__(self, resource): + self._properties = resource + + @property + def session_id(self) -> Optional[str]: + """The ID of the session.""" + return self._properties.get("sessionId") + + class UnknownJob(_AsyncJob): """A job whose type cannot be determined.""" @@ -1005,7 +1036,9 @@ def from_api_repr(cls, resource: dict, client) -> "UnknownJob": Returns: UnknownJob: Job corresponding to the resource. """ - job_ref_properties = resource.get("jobReference", {"projectId": client.project}) + job_ref_properties = resource.get( + "jobReference", {"projectId": client.project, "jobId": None} + ) job_ref = _JobReference._from_api_repr(job_ref_properties) job = cls(job_ref, client) # Populate the job reference with the project, even if it has been diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 0cb4798be..942c85fc3 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -18,7 +18,7 @@ import copy import re import typing -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Iterable, List, Optional, Union from google.api_core import exceptions from google.api_core.future import polling as polling_future @@ -31,13 +31,17 @@ from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery import _helpers -from google.cloud.bigquery.query import _query_param_from_api_repr -from google.cloud.bigquery.query import ArrayQueryParameter -from google.cloud.bigquery.query import ScalarQueryParameter -from google.cloud.bigquery.query import StructQueryParameter -from google.cloud.bigquery.query import UDFResource +from google.cloud.bigquery.query import ( + _query_param_from_api_repr, + ArrayQueryParameter, + ConnectionProperty, + ScalarQueryParameter, + StructQueryParameter, + UDFResource, +) from google.cloud.bigquery.retry import DEFAULT_RETRY, DEFAULT_JOB_RETRY from google.cloud.bigquery.routine import RoutineReference +from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import _EmptyRowIterator from google.cloud.bigquery.table import RangePartitioning from google.cloud.bigquery.table import _table_arg_to_table_ref @@ -57,6 +61,7 @@ import pyarrow from google.api_core import retry as retries from google.cloud import bigquery_storage + from google.cloud.bigquery.client import Client from google.cloud.bigquery.table import RowIterator @@ -267,6 +272,24 @@ def allow_large_results(self): def allow_large_results(self, value): self._set_sub_prop("allowLargeResults", value) + @property + def connection_properties(self) -> List[ConnectionProperty]: + """Connection properties. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery.FIELDS.connection_properties + + .. versionadded:: 2.29.0 + """ + resource = self._get_sub_prop("connectionProperties", []) + return [ConnectionProperty.from_api_repr(prop) for prop in resource] + + @connection_properties.setter + def connection_properties(self, value: Iterable[ConnectionProperty]): + self._set_sub_prop( + "connectionProperties", [prop.to_api_repr() for prop in value], + ) + @property def create_disposition(self): """google.cloud.bigquery.job.CreateDisposition: Specifies behavior @@ -281,6 +304,27 @@ def create_disposition(self): def create_disposition(self, value): self._set_sub_prop("createDisposition", value) + @property + def create_session(self) -> Optional[bool]: + """[Preview] If :data:`True`, creates a new session, where + :attr:`~google.cloud.bigquery.job.QueryJob.session_info` will contain a + random server generated session id. + + If :data:`False`, runs query with an existing ``session_id`` passed in + :attr:`~google.cloud.bigquery.job.QueryJobConfig.connection_properties`, + otherwise runs query in non-session mode. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery.FIELDS.create_session + + .. versionadded:: 2.29.0 + """ + return self._get_sub_prop("createSession") + + @create_session.setter + def create_session(self, value: Optional[bool]): + self._set_sub_prop("createSession", value) + @property def default_dataset(self): """google.cloud.bigquery.dataset.DatasetReference: the default dataset @@ -611,7 +655,7 @@ def schema_update_options(self, values): @property def script_options(self) -> ScriptOptions: - """Connection properties which can modify the query behavior. + """Options controlling the execution of scripts. https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#scriptoptions """ @@ -692,6 +736,15 @@ def allow_large_results(self): """ return self._configuration.allow_large_results + @property + def connection_properties(self) -> List[ConnectionProperty]: + """See + :attr:`google.cloud.bigquery.job.QueryJobConfig.connection_properties`. + + .. versionadded:: 2.29.0 + """ + return self._configuration.connection_properties + @property def create_disposition(self): """See @@ -699,6 +752,15 @@ def create_disposition(self): """ return self._configuration.create_disposition + @property + def create_session(self) -> Optional[bool]: + """See + :attr:`google.cloud.bigquery.job.QueryJobConfig.create_session`. + + .. versionadded:: 2.29.0 + """ + return self._configuration.create_session + @property def default_dataset(self): """See @@ -853,7 +915,7 @@ def to_api_repr(self): } @classmethod - def from_api_repr(cls, resource: dict, client) -> "QueryJob": + def from_api_repr(cls, resource: dict, client: "Client") -> "QueryJob": """Factory: construct a job given its API representation Args: @@ -866,8 +928,10 @@ def from_api_repr(cls, resource: dict, client) -> "QueryJob": Returns: google.cloud.bigquery.job.QueryJob: Job parsed from ``resource``. """ - cls._check_resource_config(resource) - job_ref = _JobReference._from_api_repr(resource["jobReference"]) + job_ref_properties = resource.setdefault( + "jobReference", {"projectId": client.project, "jobId": None} + ) + job_ref = _JobReference._from_api_repr(job_ref_properties) job = cls(job_ref, None, client=client) job._set_properties(resource) return job @@ -887,6 +951,18 @@ def query_plan(self): plan_entries = self._job_statistics().get("queryPlan", ()) return [QueryPlanEntry.from_api_repr(entry) for entry in plan_entries] + @property + def schema(self) -> Optional[List[SchemaField]]: + """The schema of the results. + + Present only for successful dry run of non-legacy SQL queries. + """ + resource = self._job_statistics().get("schema") + if resource is None: + return None + fields = resource.get("fields", []) + return [SchemaField.from_api_repr(field) for field in fields] + @property def timeline(self): """List(TimelineEntry): Return the query execution timeline @@ -1318,6 +1394,8 @@ def result( If Non-``None`` and non-default ``job_retry`` is provided and the job is not retryable. """ + if self.dry_run: + return _EmptyRowIterator() try: retry_do_query = getattr(self, "_retry_do_query", None) if retry_do_query is not None: diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 1f449f189..708f5f47b 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -18,7 +18,7 @@ import copy import datetime import decimal -from typing import Optional, Union +from typing import Any, Optional, Dict, Union from google.cloud.bigquery.table import _parse_schema_resource from google.cloud.bigquery._helpers import _rows_from_json @@ -31,6 +31,65 @@ ] +class ConnectionProperty: + """A connection-level property to customize query behavior. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/ConnectionProperty + + Args: + key: + The key of the property to set, for example, ``'time_zone'`` or + ``'session_id'``. + value: The value of the property to set. + """ + + def __init__(self, key: str = "", value: str = ""): + self._properties = { + "key": key, + "value": value, + } + + @property + def key(self) -> str: + """Name of the property. + + For example: + + * ``time_zone`` + * ``session_id`` + """ + return self._properties["key"] + + @property + def value(self) -> str: + """Value of the property.""" + return self._properties["value"] + + @classmethod + def from_api_repr(cls, resource) -> "ConnectionProperty": + """Construct :class:`~google.cloud.bigquery.query.ConnectionProperty` + from JSON resource. + + Args: + resource: JSON representation. + + Returns: + A connection property. + """ + value = cls() + value._properties = resource + return value + + def to_api_repr(self) -> Dict[str, Any]: + """Construct JSON API representation for the connection property. + + Returns: + JSON mapping + """ + return self._properties + + class UDFResource(object): """Describe a single user-defined function (UDF) resource. diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 967959b05..c8ba30be0 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.28.1" +__version__ = "2.29.0" diff --git a/google/cloud/bigquery_v2/types/encryption_config.py b/google/cloud/bigquery_v2/types/encryption_config.py index 4b9139733..a95954a30 100644 --- a/google/cloud/bigquery_v2/types/encryption_config.py +++ b/google/cloud/bigquery_v2/types/encryption_config.py @@ -25,6 +25,7 @@ class EncryptionConfiguration(proto.Message): r""" + Attributes: kms_key_name (google.protobuf.wrappers_pb2.StringValue): Optional. Describes the Cloud KMS encryption diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index 706418401..6e3ca0095 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -38,6 +38,7 @@ class Model(proto.Message): r""" + Attributes: etag (str): Output only. A hash of this resource. @@ -251,7 +252,8 @@ class FeedbackType(proto.Enum): EXPLICIT = 2 class SeasonalPeriod(proto.Message): - r""" """ + r""" + """ class SeasonalPeriodType(proto.Enum): r"""""" @@ -264,7 +266,8 @@ class SeasonalPeriodType(proto.Enum): YEARLY = 6 class KmeansEnums(proto.Message): - r""" """ + r""" + """ class KmeansInitializationMethod(proto.Enum): r"""Indicates the method used to initialize the centroids for @@ -386,6 +389,7 @@ class BinaryClassificationMetrics(proto.Message): class BinaryConfusionMatrix(proto.Message): r"""Confusion matrix for binary classification models. + Attributes: positive_class_threshold (google.protobuf.wrappers_pb2.DoubleValue): Threshold value used when computing each of @@ -464,6 +468,7 @@ class MultiClassClassificationMetrics(proto.Message): class ConfusionMatrix(proto.Message): r"""Confusion matrix for multi-class classification models. + Attributes: confidence_threshold (google.protobuf.wrappers_pb2.DoubleValue): Confidence threshold used when computing the @@ -474,6 +479,7 @@ class ConfusionMatrix(proto.Message): class Entry(proto.Message): r"""A single entry in the confusion matrix. + Attributes: predicted_label (str): The predicted label. For confidence_threshold > 0, we will @@ -491,6 +497,7 @@ class Entry(proto.Message): class Row(proto.Message): r"""A single row in the confusion matrix. + Attributes: actual_label (str): The original label of this row. @@ -525,6 +532,7 @@ class Row(proto.Message): class ClusteringMetrics(proto.Message): r"""Evaluation metrics for clustering models. + Attributes: davies_bouldin_index (google.protobuf.wrappers_pb2.DoubleValue): Davies-Bouldin index. @@ -537,6 +545,7 @@ class ClusteringMetrics(proto.Message): class Cluster(proto.Message): r"""Message containing the information about one cluster. + Attributes: centroid_id (int): Centroid id. @@ -550,6 +559,7 @@ class Cluster(proto.Message): class FeatureValue(proto.Message): r"""Representative value of a single feature within the cluster. + Attributes: feature_column (str): The feature column name. @@ -562,6 +572,7 @@ class FeatureValue(proto.Message): class CategoricalValue(proto.Message): r"""Representative value of a categorical feature. + Attributes: category_counts (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount]): Counts of all categories for the categorical feature. If @@ -573,6 +584,7 @@ class CategoricalValue(proto.Message): class CategoryCount(proto.Message): r"""Represents the count of a single category within the cluster. + Attributes: category (str): The name of category. @@ -668,6 +680,7 @@ class RankingMetrics(proto.Message): class ArimaForecastingMetrics(proto.Message): r"""Model evaluation metrics for ARIMA forecasting models. + Attributes: non_seasonal_order (Sequence[google.cloud.bigquery_v2.types.Model.ArimaOrder]): Non-seasonal order. @@ -857,6 +870,7 @@ class ArimaOrder(proto.Message): class ArimaFittingMetrics(proto.Message): r"""ARIMA model fitting metrics. + Attributes: log_likelihood (float): Log-likelihood. @@ -888,6 +902,7 @@ class GlobalExplanation(proto.Message): class Explanation(proto.Message): r"""Explanation for a single feature. + Attributes: feature_name (str): Full name of the feature. For non-numerical features, will @@ -910,6 +925,7 @@ class Explanation(proto.Message): class TrainingRun(proto.Message): r"""Information about a single training query run for the model. + Attributes: training_options (google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions): Options that were used for this training run, @@ -935,6 +951,7 @@ class TrainingRun(proto.Message): class TrainingOptions(proto.Message): r"""Options used in model training. + Attributes: max_iterations (int): The maximum number of iterations in training. @@ -1182,6 +1199,7 @@ class TrainingOptions(proto.Message): class IterationResult(proto.Message): r"""Information about a single iteration of the training run. + Attributes: index (google.protobuf.wrappers_pb2.Int32Value): Index of the iteration, 0 based. @@ -1205,6 +1223,7 @@ class IterationResult(proto.Message): class ClusterInfo(proto.Message): r"""Information about a single cluster for clustering model. + Attributes: centroid_id (int): Centroid id. @@ -1241,6 +1260,7 @@ class ArimaResult(proto.Message): class ArimaCoefficients(proto.Message): r"""Arima coefficients. + Attributes: auto_regressive_coefficients (Sequence[float]): Auto-regressive coefficients, an array of @@ -1263,6 +1283,7 @@ class ArimaCoefficients(proto.Message): class ArimaModelInfo(proto.Message): r"""Arima model information. + Attributes: non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): Non-seasonal order. @@ -1409,6 +1430,7 @@ class ArimaModelInfo(proto.Message): class GetModelRequest(proto.Message): r""" + Attributes: project_id (str): Required. Project ID of the requested model. @@ -1425,6 +1447,7 @@ class GetModelRequest(proto.Message): class PatchModelRequest(proto.Message): r""" + Attributes: project_id (str): Required. Project ID of the model to patch. @@ -1447,6 +1470,7 @@ class PatchModelRequest(proto.Message): class DeleteModelRequest(proto.Message): r""" + Attributes: project_id (str): Required. Project ID of the model to delete. @@ -1463,6 +1487,7 @@ class DeleteModelRequest(proto.Message): class ListModelsRequest(proto.Message): r""" + Attributes: project_id (str): Required. Project ID of the models to list. @@ -1487,6 +1512,7 @@ class ListModelsRequest(proto.Message): class ListModelsResponse(proto.Message): r""" + Attributes: models (Sequence[google.cloud.bigquery_v2.types.Model]): Models in the requested dataset. Only the following fields diff --git a/google/cloud/bigquery_v2/types/model_reference.py b/google/cloud/bigquery_v2/types/model_reference.py index a9ebad613..544377f61 100644 --- a/google/cloud/bigquery_v2/types/model_reference.py +++ b/google/cloud/bigquery_v2/types/model_reference.py @@ -23,6 +23,7 @@ class ModelReference(proto.Message): r"""Id path of a model. + Attributes: project_id (str): Required. The ID of the project containing diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py index 7a845fc48..69a221c3c 100644 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ b/google/cloud/bigquery_v2/types/standard_sql.py @@ -78,6 +78,7 @@ class TypeKind(proto.Enum): class StandardSqlField(proto.Message): r"""A field or a column. + Attributes: name (str): Optional. The name of this field. Can be @@ -96,6 +97,7 @@ class StandardSqlField(proto.Message): class StandardSqlStructType(proto.Message): r""" + Attributes: fields (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): @@ -106,6 +108,7 @@ class StandardSqlStructType(proto.Message): class StandardSqlTableType(proto.Message): r"""A table type + Attributes: columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): The columns in this table type diff --git a/google/cloud/bigquery_v2/types/table_reference.py b/google/cloud/bigquery_v2/types/table_reference.py index d56e5b09f..da206b4d7 100644 --- a/google/cloud/bigquery_v2/types/table_reference.py +++ b/google/cloud/bigquery_v2/types/table_reference.py @@ -23,6 +23,7 @@ class TableReference(proto.Message): r""" + Attributes: project_id (str): Required. The ID of the project containing diff --git a/owlbot.py b/owlbot.py index 0f6f8fe99..f2f8bea54 100644 --- a/owlbot.py +++ b/owlbot.py @@ -98,8 +98,9 @@ microgenerator=True, split_system_tests=True, intersphinx_dependencies={ - "pandas": "http://pandas.pydata.org/pandas-docs/stable/", + "dateutil": "https://dateutil.readthedocs.io/en/latest/", "geopandas": "https://geopandas.org/", + "pandas": "https://pandas.pydata.org/pandas-docs/dev", }, ) @@ -115,10 +116,6 @@ # Include custom SNIPPETS_TESTS job for performance. # https://github.com/googleapis/python-bigquery/issues/191 ".kokoro/presubmit/presubmit.cfg", - # Group all renovate PRs together. If this works well, remove this and - # update the shared templates (possibly with configuration option to - # py_library.) - "renovate.json", ], ) diff --git a/renovate.json b/renovate.json index 713c60bb4..c21036d38 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,9 @@ { "extends": [ - "config:base", "group:all", ":preserveSemverRanges" + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" ], "ignorePaths": [".pre-commit-config.yaml"], "pip_requirements": { diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 1fd8956fb..93a9122cc 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 1fd8956fb..93a9122cc 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/setup.py b/setup.py index e7515493d..95dad190a 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" +pyarrow_dep = ["pyarrow >= 3.0.0, < 7.0dev"] dependencies = [ "grpcio >= 1.38.1, < 2.0dev", # https://github.com/googleapis/python-bigquery/issues/695 # NOTE: Maintainers, please do not require google-api-core>=2.x.x @@ -42,6 +43,7 @@ "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3", "protobuf >= 3.12.0", + "python-dateutil >= 2.7.2, <3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { @@ -54,11 +56,11 @@ # grpc.Channel.close() method isn't added until 1.32.0. # https://github.com/grpc/grpc/pull/15254 "grpcio >= 1.38.1, < 2.0dev", - "pyarrow >= 3.0.0, < 6.0dev", - ], + ] + + pyarrow_dep, "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"], - "pandas": ["pandas>=0.23.0", "pyarrow >= 3.0.0, < 6.0dev"], - "bignumeric_type": ["pyarrow >= 3.0.0, < 6.0dev"], + "pandas": ["pandas>=0.23.0"] + pyarrow_dep, + "bignumeric_type": pyarrow_dep, "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"], "opentelemetry": [ "opentelemetry-api >= 0.11b0", diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 23d2724f7..59913d588 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -18,6 +18,7 @@ pandas==0.24.2 proto-plus==1.10.0 protobuf==3.12.0 pyarrow==3.0.0 +python-dateutil==2.7.2 requests==2.18.0 Shapely==1.6.0 six==1.13.0 diff --git a/tests/system/test_client.py b/tests/system/test_client.py index f6f95c184..91bcff155 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -37,11 +37,6 @@ except ImportError: # pragma: NO COVER bigquery_storage = None -try: - import fastavro # to parse BQ storage client results -except ImportError: # pragma: NO COVER - fastavro = None - try: import pyarrow import pyarrow.types diff --git a/tests/system/test_list_rows.py b/tests/system/test_list_rows.py index 70388059e..4c08958c3 100644 --- a/tests/system/test_list_rows.py +++ b/tests/system/test_list_rows.py @@ -15,6 +15,8 @@ import datetime import decimal +from dateutil import relativedelta + from google.cloud import bigquery from google.cloud.bigquery import enums @@ -64,6 +66,9 @@ def test_list_rows_scalars(bigquery_client: bigquery.Client, scalars_table: str) assert row["datetime_col"] == datetime.datetime(2021, 7, 21, 11, 39, 45) assert row["geography_col"] == "POINT(-122.0838511 37.3860517)" assert row["int64_col"] == 123456789 + assert row["interval_col"] == relativedelta.relativedelta( + years=7, months=11, days=9, hours=4, minutes=15, seconds=37, microseconds=123456 + ) assert row["numeric_col"] == decimal.Decimal("1.23456789") assert row["bignumeric_col"] == decimal.Decimal("10.111213141516171819") assert row["float64_col"] == 1.25 @@ -95,6 +100,9 @@ def test_list_rows_scalars_extreme( assert row["datetime_col"] == datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) assert row["geography_col"] == "POINT(-135 90)" assert row["int64_col"] == 9223372036854775807 + assert row["interval_col"] == relativedelta.relativedelta( + years=-10000, days=-3660000, hours=-87840000 + ) assert row["numeric_col"] == decimal.Decimal(f"9.{'9' * 37}E+28") assert row["bignumeric_col"] == decimal.Decimal(f"9.{'9' * 75}E+37") assert row["float64_col"] == float("Inf") diff --git a/tests/system/test_query.py b/tests/system/test_query.py new file mode 100644 index 000000000..649120a7e --- /dev/null +++ b/tests/system/test_query.py @@ -0,0 +1,55 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery + + +def test_dry_run(bigquery_client: bigquery.Client, scalars_table: str): + query_config = bigquery.QueryJobConfig() + query_config.dry_run = True + + query_string = f"SELECT * FROM {scalars_table}" + query_job = bigquery_client.query(query_string, job_config=query_config,) + + # Note: `query_job.result()` is not necessary on a dry run query. All + # necessary information is returned in the initial response. + assert query_job.dry_run is True + assert query_job.total_bytes_processed > 0 + assert len(query_job.schema) > 0 + + +def test_session(bigquery_client: bigquery.Client): + initial_config = bigquery.QueryJobConfig() + initial_config.create_session = True + initial_query = """ + CREATE TEMPORARY TABLE numbers(id INT64) + AS + SELECT * FROM UNNEST([1, 2, 3, 4, 5]) AS id; + """ + initial_job = bigquery_client.query(initial_query, job_config=initial_config) + initial_job.result() + session_id = initial_job.session_info.session_id + assert session_id is not None + + second_config = bigquery.QueryJobConfig() + second_config.connection_properties = [ + bigquery.ConnectionProperty("session_id", session_id), + ] + second_job = bigquery_client.query( + "SELECT COUNT(*) FROM numbers;", job_config=second_config + ) + rows = list(second_job.result()) + + assert len(rows) == 1 + assert rows[0][0] == 5 diff --git a/tests/unit/helpers/test_from_json.py b/tests/unit/helpers/test_from_json.py new file mode 100644 index 000000000..65b054f44 --- /dev/null +++ b/tests/unit/helpers/test_from_json.py @@ -0,0 +1,157 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dateutil.relativedelta import relativedelta +import pytest + +from google.cloud.bigquery.schema import SchemaField + + +def create_field(mode="NULLABLE", type_="IGNORED"): + return SchemaField("test_field", type_, mode=mode) + + +@pytest.fixture +def mut(): + from google.cloud.bigquery import _helpers + + return _helpers + + +def test_interval_from_json_w_none_nullable(mut): + got = mut._interval_from_json(None, create_field()) + assert got is None + + +def test_interval_from_json_w_none_required(mut): + with pytest.raises(TypeError): + mut._interval_from_json(None, create_field(mode="REQUIRED")) + + +def test_interval_from_json_w_invalid_format(mut): + with pytest.raises(ValueError, match="NOT_AN_INTERVAL"): + mut._interval_from_json("NOT_AN_INTERVAL", create_field()) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ("0-0 0 0:0:0", relativedelta()), + # SELECT INTERVAL X YEAR + ("-10000-0 0 0:0:0", relativedelta(years=-10000)), + ("-1-0 0 0:0:0", relativedelta(years=-1)), + ("1-0 0 0:0:0", relativedelta(years=1)), + ("10000-0 0 0:0:0", relativedelta(years=10000)), + # SELECT INTERVAL X MONTH + ("-0-11 0 0:0:0", relativedelta(months=-11)), + ("-0-1 0 0:0:0", relativedelta(months=-1)), + ("0-1 0 0:0:0", relativedelta(months=1)), + ("0-11 0 0:0:0", relativedelta(months=11)), + # SELECT INTERVAL X DAY + ("0-0 -3660000 0:0:0", relativedelta(days=-3660000)), + ("0-0 -1 0:0:0", relativedelta(days=-1)), + ("0-0 1 0:0:0", relativedelta(days=1)), + ("0-0 3660000 0:0:0", relativedelta(days=3660000)), + # SELECT INTERVAL X HOUR + ("0-0 0 -87840000:0:0", relativedelta(hours=-87840000)), + ("0-0 0 -1:0:0", relativedelta(hours=-1)), + ("0-0 0 1:0:0", relativedelta(hours=1)), + ("0-0 0 87840000:0:0", relativedelta(hours=87840000)), + # SELECT INTERVAL X MINUTE + ("0-0 0 -0:59:0", relativedelta(minutes=-59)), + ("0-0 0 -0:1:0", relativedelta(minutes=-1)), + ("0-0 0 0:1:0", relativedelta(minutes=1)), + ("0-0 0 0:59:0", relativedelta(minutes=59)), + # SELECT INTERVAL X SECOND + ("0-0 0 -0:0:59", relativedelta(seconds=-59)), + ("0-0 0 -0:0:1", relativedelta(seconds=-1)), + ("0-0 0 0:0:1", relativedelta(seconds=1)), + ("0-0 0 0:0:59", relativedelta(seconds=59)), + # SELECT (INTERVAL -1 SECOND) / 1000000 + ("0-0 0 -0:0:0.000001", relativedelta(microseconds=-1)), + ("0-0 0 -0:0:59.999999", relativedelta(seconds=-59, microseconds=-999999)), + ("0-0 0 -0:0:59.999", relativedelta(seconds=-59, microseconds=-999000)), + ("0-0 0 0:0:59.999", relativedelta(seconds=59, microseconds=999000)), + ("0-0 0 0:0:59.999999", relativedelta(seconds=59, microseconds=999999)), + # Test with multiple digits in each section. + ( + "32-11 45 67:16:23.987654", + relativedelta( + years=32, + months=11, + days=45, + hours=67, + minutes=16, + seconds=23, + microseconds=987654, + ), + ), + ( + "-32-11 -45 -67:16:23.987654", + relativedelta( + years=-32, + months=-11, + days=-45, + hours=-67, + minutes=-16, + seconds=-23, + microseconds=-987654, + ), + ), + # Test with mixed +/- sections. + ( + "9999-9 -999999 9999999:59:59.999999", + relativedelta( + years=9999, + months=9, + days=-999999, + hours=9999999, + minutes=59, + seconds=59, + microseconds=999999, + ), + ), + # Test with fraction that is not microseconds. + ("0-0 0 0:0:42.", relativedelta(seconds=42)), + ("0-0 0 0:0:59.1", relativedelta(seconds=59, microseconds=100000)), + ("0-0 0 0:0:0.12", relativedelta(microseconds=120000)), + ("0-0 0 0:0:0.123", relativedelta(microseconds=123000)), + ("0-0 0 0:0:0.1234", relativedelta(microseconds=123400)), + # Fractional seconds can cause rounding problems if cast to float. See: + # https://github.com/googleapis/python-db-dtypes-pandas/issues/18 + ("0-0 0 0:0:59.876543", relativedelta(seconds=59, microseconds=876543)), + ( + "0-0 0 01:01:01.010101", + relativedelta(hours=1, minutes=1, seconds=1, microseconds=10101), + ), + ( + "0-0 0 09:09:09.090909", + relativedelta(hours=9, minutes=9, seconds=9, microseconds=90909), + ), + ( + "0-0 0 11:11:11.111111", + relativedelta(hours=11, minutes=11, seconds=11, microseconds=111111), + ), + ( + "0-0 0 19:16:23.987654", + relativedelta(hours=19, minutes=16, seconds=23, microseconds=987654), + ), + # Nanoseconds are not expected, but should not cause error. + ("0-0 0 0:0:00.123456789", relativedelta(microseconds=123456)), + ("0-0 0 0:0:59.87654321", relativedelta(seconds=59, microseconds=876543)), + ), +) +def test_w_string_values(mut, value, expected): + got = mut._interval_from_json(value, create_field()) + assert got == expected diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index e320c72cb..250be83bb 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -228,6 +228,15 @@ def test_script_statistics(self): self.assertEqual(stack_frame.end_column, 14) self.assertEqual(stack_frame.text, "QUERY TEXT") + def test_session_info(self): + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, client) + + self.assertIsNone(job.session_info) + job._properties["statistics"] = {"sessionInfo": {"sessionId": "abcdefg"}} + self.assertIsNotNone(job.session_info) + self.assertEqual(job.session_info.session_id, "abcdefg") + def test_transaction_info(self): from google.cloud.bigquery.job.base import TransactionInfo diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 4c598d797..4da035b78 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -26,6 +26,7 @@ from google.cloud.bigquery.client import _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS import google.cloud.bigquery.query +from google.cloud.bigquery.table import _EmptyRowIterator from ..helpers import make_connection @@ -268,25 +269,6 @@ def test_ctor_w_query_parameters(self): job = self._make_one(self.JOB_ID, self.QUERY, client, job_config=config) self.assertEqual(job.query_parameters, query_parameters) - def test_from_api_repr_missing_identity(self): - self._setUpConstants() - client = _make_client(project=self.PROJECT) - RESOURCE = {} - klass = self._get_target_class() - with self.assertRaises(KeyError): - klass.from_api_repr(RESOURCE, client=client) - - def test_from_api_repr_missing_config(self): - self._setUpConstants() - client = _make_client(project=self.PROJECT) - RESOURCE = { - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "jobReference": {"projectId": self.PROJECT, "jobId": self.JOB_ID}, - } - klass = self._get_target_class() - with self.assertRaises(KeyError): - klass.from_api_repr(RESOURCE, client=client) - def test_from_api_repr_bare(self): self._setUpConstants() client = _make_client(project=self.PROJECT) @@ -299,6 +281,8 @@ def test_from_api_repr_bare(self): job = klass.from_api_repr(RESOURCE, client=client) self.assertIs(job._client, client) self._verifyResourceProperties(job, RESOURCE) + self.assertEqual(len(job.connection_properties), 0) + self.assertIsNone(job.create_session) def test_from_api_repr_with_encryption(self): self._setUpConstants() @@ -989,6 +973,19 @@ def test_result(self): [query_results_call, query_results_call, reload_call, query_page_call] ) + def test_result_dry_run(self): + job_resource = self._make_resource(started=True, location="EU") + job_resource["configuration"]["dryRun"] = True + conn = make_connection() + client = _make_client(self.PROJECT, connection=conn) + job = self._get_target_class().from_api_repr(job_resource, client) + + result = job.result() + + calls = conn.api_request.mock_calls + self.assertIsInstance(result, _EmptyRowIterator) + self.assertEqual(calls, []) + def test_result_with_done_job_calls_get_query_results(self): query_resource_done = { "jobComplete": True, @@ -1391,6 +1388,43 @@ def test_result_transport_timeout_error(self): with call_api_patch, self.assertRaises(concurrent.futures.TimeoutError): job.result(timeout=1) + def test_no_schema(self): + client = _make_client(project=self.PROJECT) + resource = {} + klass = self._get_target_class() + job = klass.from_api_repr(resource, client=client) + assert job.schema is None + + def test_schema(self): + client = _make_client(project=self.PROJECT) + resource = { + "statistics": { + "query": { + "schema": { + "fields": [ + {"mode": "NULLABLE", "name": "bool_col", "type": "BOOLEAN"}, + { + "mode": "NULLABLE", + "name": "string_col", + "type": "STRING", + }, + { + "mode": "NULLABLE", + "name": "timestamp_col", + "type": "TIMESTAMP", + }, + ] + }, + }, + }, + } + klass = self._get_target_class() + job = klass.from_api_repr(resource, client=client) + assert len(job.schema) == 3 + assert job.schema[0].field_type == "BOOLEAN" + assert job.schema[1].field_type == "STRING" + assert job.schema[2].field_type == "TIMESTAMP" + def test__begin_error(self): from google.cloud import exceptions diff --git a/tests/unit/job/test_query_config.py b/tests/unit/job/test_query_config.py index 109cf7e44..7818236f4 100644 --- a/tests/unit/job/test_query_config.py +++ b/tests/unit/job/test_query_config.py @@ -152,6 +152,27 @@ def test_clustering_fields(self): config.clustering_fields = None self.assertIsNone(config.clustering_fields) + def test_connection_properties(self): + from google.cloud.bigquery.job.query import ConnectionProperty + + config = self._get_target_class()() + self.assertEqual(len(config.connection_properties), 0) + + session_id = ConnectionProperty("session_id", "abcd") + time_zone = ConnectionProperty("time_zone", "America/Chicago") + config.connection_properties = [session_id, time_zone] + self.assertEqual(len(config.connection_properties), 2) + self.assertEqual(config.connection_properties[0].key, "session_id") + self.assertEqual(config.connection_properties[0].value, "abcd") + self.assertEqual(config.connection_properties[1].key, "time_zone") + self.assertEqual(config.connection_properties[1].value, "America/Chicago") + + def test_create_session(self): + config = self._get_target_class()() + self.assertIsNone(config.create_session) + config.create_session = True + self.assertTrue(config.create_session) + def test_from_api_repr_empty(self): klass = self._get_target_class() config = klass.from_api_repr({})