Skip to content

Commit e1cd422

Browse files
kojiromikekojiromike
committed
AVRO-2921: Strict Type Checking
1 parent caa3a42 commit e1cd422

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+2353
-159
lines changed

‎lang/py/avro/codecs.pyi

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env python3
2+
# -*- mode: python -*-
3+
# -*- coding: utf-8 -*-
4+
5+
##
6+
# Licensed to the Apache Software Foundation (ASF) under one
7+
# or more contributor license agreements. See the NOTICE file
8+
# distributed with this work for additional information
9+
# regarding copyright ownership. The ASF licenses this file
10+
# to you under the Apache License, Version 2.0 (the
11+
# "License"); you may not use this file except in compliance
12+
# with the License. You may obtain a copy of the License at
13+
#
14+
# https://www.apache.org/licenses/LICENSE-2.0
15+
#
16+
# Unless required by applicable law or agreed to in writing, software
17+
# distributed under the License is distributed on an "AS IS" BASIS,
18+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19+
# See the License for the specific language governing permissions and
20+
# limitations under the License.
21+
22+
"""
23+
Contains Codecs for Python Avro.
24+
25+
Note that the word "codecs" means "compression/decompression algorithms" in the
26+
Avro world (https://avro.apache.org/docs/current/spec.html#Object+Container+Files),
27+
so don't confuse it with the Python's "codecs", which is a package mainly for
28+
converting charsets (https://docs.python.org/3/library/codecs.html).
29+
"""
30+
31+
import abc
32+
from typing import List
33+
34+
import avro.io
35+
36+
class Codec(abc.ABC):
37+
"""Abstract base class for all Avro codec classes."""
38+
39+
def compress(self, data: bytes) -> bytes:
40+
...
41+
42+
def decompress(self, readers_decoder: avro.io.BinaryDecoder) -> avro.io.BinaryDecoder:
43+
...
44+
45+
46+
class SnappyCodec(Codec):
47+
48+
def check_crc32(self, bytes_: bytes, checksum: bytes) -> None:
49+
...
50+
51+
52+
def get_codec(codec_name: str) -> Codec:
53+
...
54+
55+
56+
def supported_codec_names() -> List[str]:
57+
...

‎lang/py/avro/datafile.pyi

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#!/usr/bin/env python3
2+
# -*- mode: python -*-
3+
# -*- coding: utf-8 -*-
4+
5+
##
6+
# Licensed to the Apache Software Foundation (ASF) under one
7+
# or more contributor license agreements. See the NOTICE file
8+
# distributed with this work for additional information
9+
# regarding copyright ownership. The ASF licenses this file
10+
# to you under the Apache License, Version 2.0 (the
11+
# "License"); you may not use this file except in compliance
12+
# with the License. You may obtain a copy of the License at
13+
#
14+
# https://www.apache.org/licenses/LICENSE-2.0
15+
#
16+
# Unless required by applicable law or agreed to in writing, software
17+
# distributed under the License is distributed on an "AS IS" BASIS,
18+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19+
# See the License for the specific language governing permissions and
20+
# limitations under the License.
21+
22+
from types import TracebackType
23+
from typing import *
24+
25+
from avro.io import BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter
26+
from avro.schema import Schema
27+
from avro.types import AvroAny
28+
29+
VALID_CODECS = () # type: Sequence[str]
30+
31+
32+
class _DataFile:
33+
def __exit__(self, type: Type[BaseException], value: BaseException, traceback: TracebackType) -> None:
34+
...
35+
def get_meta(self, key: str) -> bytes:
36+
...
37+
def set_meta(self, key: str, val: bytes) -> None:
38+
...
39+
@property
40+
def sync_marker(self) -> int:
41+
...
42+
@property
43+
def meta(self) -> Dict[str, bytes]:
44+
...
45+
@property
46+
def codec(self) -> str:
47+
...
48+
@codec.setter
49+
def codec(self, value: str) -> None:
50+
...
51+
@property
52+
def schema(self) -> Schema:
53+
...
54+
@schema.setter
55+
def schema(self, value: Schema) -> None:
56+
...
57+
58+
59+
class DataFileWriter(_DataFile):
60+
def __init__(self, writer: BinaryIO, datum_writer: DatumWriter, writers_schema: Optional[Schema], codec: str) -> None:
61+
...
62+
def __enter__(self) -> "DataFileWriter":
63+
...
64+
@property
65+
def writer(self) -> BinaryIO:
66+
...
67+
@property
68+
def encoder(self) -> BinaryEncoder:
69+
...
70+
@property
71+
def datum_writer(self) -> DatumWriter:
72+
...
73+
@property
74+
def buffer_writer(self) -> BinaryIO:
75+
...
76+
@property
77+
def buffer_encoder(self) -> BinaryEncoder:
78+
...
79+
def _write_header(self) -> None:
80+
...
81+
def _write_block(self) -> None:
82+
...
83+
def append(self, datum: AvroAny) -> None:
84+
...
85+
def sync(self) -> int:
86+
...
87+
def flush(self) -> None:
88+
...
89+
def close(self) -> None:
90+
...
91+
92+
class DataFileReader(_DataFile):
93+
def __init__(self, reader: BinaryIO, datum_reader: DatumReader) -> None:
94+
...
95+
def __iter__(self) -> "DataFileReader":
96+
...
97+
def __enter__(self) -> "DataFileReader":
98+
...
99+
@property
100+
def reader(self) -> BinaryIO:
101+
...
102+
@property
103+
def raw_decoder(self) -> BinaryDecoder:
104+
...
105+
@property
106+
def datum_decoder(self) -> BinaryDecoder:
107+
...
108+
@property
109+
def datum_reader(self) -> DatumReader:
110+
...
111+
@property
112+
def file_length(self) -> int:
113+
...
114+
def determine_file_length(self) -> int:
115+
...
116+
def is_EOF(self) -> bool:
117+
...
118+
def _read_header(self) -> None:
119+
...
120+
def _read_block_header(self) -> None:
121+
...
122+
def _skip_sync(self) -> bool:
123+
...
124+
def __next__(self) -> AvroAny:
125+
...
126+
def close(self) -> None:
127+
...
128+
129+
def generate_sixteen_random_bytes() -> bytes:
130+
...

‎lang/py/avro/errors.pyi

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env python3
2+
# -*- mode: python -*-
3+
# -*- coding: utf-8 -*-
4+
5+
##
6+
# Licensed to the Apache Software Foundation (ASF) under one
7+
# or more contributor license agreements. See the NOTICE file
8+
# distributed with this work for additional information
9+
# regarding copyright ownership. The ASF licenses this file
10+
# to you under the Apache License, Version 2.0 (the
11+
# "License"); you may not use this file except in compliance
12+
# with the License. You may obtain a copy of the License at
13+
#
14+
# https://www.apache.org/licenses/LICENSE-2.0
15+
#
16+
# Unless required by applicable law or agreed to in writing, software
17+
# distributed under the License is distributed on an "AS IS" BASIS,
18+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19+
# See the License for the specific language governing permissions and
20+
# limitations under the License.
21+
22+
from typing import Optional
23+
24+
import avro.schema
25+
import avro.types
26+
27+
class AvroException(Exception):
28+
"""The base class for exceptions in avro."""
29+
30+
31+
class SchemaParseException(AvroException):
32+
"""Raised when a schema failed to parse."""
33+
34+
35+
class InvalidName(SchemaParseException):
36+
"""User attempted to parse a schema with an invalid name."""
37+
38+
39+
class AvroWarning(UserWarning):
40+
"""Base class for warnings."""
41+
42+
43+
class IgnoredLogicalType(AvroWarning):
44+
"""Warnings for unknown or invalid logical types."""
45+
46+
47+
class AvroTypeException(AvroException):
48+
def __init__(self,
49+
expected_schema: avro.schema.Schema,
50+
datum: avro.types.AvroAny) -> None:
51+
...
52+
53+
54+
class SchemaResolutionException(AvroException):
55+
def __init__(self,
56+
fail_msg: str,
57+
writers_schema: Optional[avro.schema.Schema]=None,
58+
readers_schema: Optional[avro.schema.Schema]=None) -> None:
59+
...
60+
61+
62+
class DataFileException(AvroException):
63+
"""Raised when there's a problem reading or writing file object containers."""
64+
65+
66+
class AvroRemoteException(AvroException):
67+
"""Raised when an error message is sent by an Avro requestor or responder."""
68+
69+
70+
class ConnectionClosedException(AvroException):
71+
"""Raised when attempting IPC on a closed connection."""
72+
73+
74+
class ProtocolParseException(AvroException):
75+
"""Raised when a protocol failed to parse."""
76+
77+
78+
class UnsupportedCodec(NotImplementedError, AvroException):
79+
"""Raised when the compression named cannot be used."""
80+
81+
82+
class UsageError(RuntimeError, AvroException):
83+
"""An exception raised when incorrect arguments were passed."""

‎lang/py/avro/io.py

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -432,8 +432,9 @@ def __init__(self, writer):
432432
"""
433433
self._writer = writer
434434

435-
# read-only properties
436-
writer = property(lambda self: self._writer)
435+
@property
436+
def writer(self):
437+
return self._writer
437438

438439
def write(self, datum):
439440
"""Write an arbitrary datum."""
@@ -443,7 +444,6 @@ def write_null(self, datum):
443444
"""
444445
null is written as zero bytes
445446
"""
446-
pass
447447

448448
def write_boolean(self, datum):
449449
"""
@@ -991,11 +991,13 @@ class DatumWriter:
991991
def __init__(self, writers_schema=None):
992992
self._writers_schema = writers_schema
993993

994-
# read/write properties
995-
def set_writers_schema(self, writers_schema):
996-
self._writers_schema = writers_schema
997-
writers_schema = property(lambda self: self._writers_schema,
998-
set_writers_schema)
994+
@property
995+
def writers_schema(self):
996+
return self._writers_schema
997+
998+
@writers_schema.setter
999+
def writers_schema(self, schema):
1000+
self._writers_schema = schema
9991001

10001002
def write(self, datum, encoder):
10011003
validate(self.writers_schema, datum, raise_on_error=True)

0 commit comments

Comments
 (0)