Skip to content

Commit 2049c00

Browse files
varadhbhatnagarVarad Bhatnagar
andauthored
Raise error for incorrect JSON serialization (#7273)
Co-authored-by: Varad Bhatnagar <[email protected]>
1 parent 01f91ba commit 2049c00

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

src/datasets/io/json.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@ def write(self) -> int:
104104
if compression not in [None, "infer", "gzip", "bz2", "xz"]:
105105
raise NotImplementedError(f"`datasets` currently does not support {compression} compression")
106106

107+
if not lines and self.batch_size < self.dataset.num_rows:
108+
raise NotImplementedError(
109+
"Output JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead."
110+
)
111+
107112
if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
108113
with fsspec.open(
109114
self.path_or_buf, "wb", compression=compression, **(self.storage_options or {})

0 commit comments

Comments
 (0)