fix: set pyarrow field nullable to False for a BigQuery field in REPEATED mode (#1999)

xyloid · Linchin · web-flow · commit 5352870283ca · 2024-08-20T08:21:01.000-05:00
Co-authored-by: Lingqing Gan &lt;lingqing.gan@gmail.com&gt;
diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py
@@ -200,7 +200,7 @@ def bq_to_arrow_field(bq_field, array_type=None):
             # local NULL values. Arrow will gladly interpret these NULL values
             # as non-NULL and give you an arbitrary value. See:
             # https://github.com/googleapis/python-bigquery/issues/1692
-            nullable=True,
+            nullable=False if bq_field.mode.upper() == "REPEATED" else True,
             metadata=metadata,
         )
 
diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py
@@ -2002,6 +2002,23 @@ def test_bq_to_arrow_field_type_override(module_under_test):
     )
 
 
+@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
+def test_bq_to_arrow_field_set_repeated_nullable_false(module_under_test):
+    assert (
+        module_under_test.bq_to_arrow_field(
+            schema.SchemaField("name", "STRING", mode="REPEATED")
+        ).nullable
+        is False
+    )
+
+    assert (
+        module_under_test.bq_to_arrow_field(
+            schema.SchemaField("name", "STRING", mode="NULLABLE")
+        ).nullable
+        is True
+    )
+
+
 @pytest.mark.parametrize(
     "field_type, metadata",
     [

Original file line number	Diff line number	Diff line change
`@@ -200,7 +200,7 @@ def bq_to_arrow_field(bq_field, array_type=None):`
`200`	`200`	`# local NULL values. Arrow will gladly interpret these NULL values`
`201`	`201`	`# as non-NULL and give you an arbitrary value. See:`
`202`	`202`	`# https://github.com/googleapis/python-bigquery/issues/1692`
`203`		`- nullable=True,`
	`203`	`+ nullable=False if bq_field.mode.upper() == "REPEATED" else True,`
`204`	`204`	`metadata=metadata,`
`205`	`205`	`)`
`206`	`206`