Skip to content

Commit 493b685

Browse files
ephraimbuddyephraimbuddy
authored
Add separate example DAGs and system tests for google cloud speech (apache#8778)
1 parent 9bb91ef commit 493b685

File tree

11 files changed

+328
-125
lines changed

11 files changed

+328
-125
lines changed

‎airflow/providers/google/cloud/example_dags/example_speech_to_text.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
import os
20+
21+
from airflow import models
22+
from airflow.providers.google.cloud.operators.speech_to_text import CloudSpeechToTextRecognizeSpeechOperator
23+
from airflow.providers.google.cloud.operators.text_to_speech import CloudTextToSpeechSynthesizeOperator
24+
from airflow.utils import dates
25+
26+
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
27+
BUCKET_NAME = os.environ.get("GCP_SPEECH_TO_TEXT_TEST_BUCKET", "gcp-speech-to-text-test-bucket")
28+
29+
# [START howto_operator_speech_to_text_gcp_filename]
30+
FILENAME = "gcp-speech-test-file"
31+
# [END howto_operator_speech_to_text_gcp_filename]
32+
33+
# [START howto_operator_text_to_speech_api_arguments]
34+
INPUT = {"text": "Sample text for demo purposes"}
35+
VOICE = {"language_code": "en-US", "ssml_gender": "FEMALE"}
36+
AUDIO_CONFIG = {"audio_encoding": "LINEAR16"}
37+
# [END howto_operator_text_to_speech_api_arguments]
38+
39+
# [START howto_operator_speech_to_text_api_arguments]
40+
CONFIG = {"encoding": "LINEAR16", "language_code": "en_US"}
41+
AUDIO = {"uri": "gs://{bucket}/{object}".format(bucket=BUCKET_NAME, object=FILENAME)}
42+
# [END howto_operator_speech_to_text_api_arguments]
43+
44+
default_args = {"start_date": dates.days_ago(1)}
45+
46+
with models.DAG(
47+
"example_gcp_speech_to_text",
48+
default_args=default_args,
49+
schedule_interval=None, # Override to match your needs
50+
tags=['example'],
51+
) as dag:
52+
text_to_speech_synthesize_task = CloudTextToSpeechSynthesizeOperator(
53+
project_id=GCP_PROJECT_ID,
54+
input_data=INPUT,
55+
voice=VOICE,
56+
audio_config=AUDIO_CONFIG,
57+
target_bucket_name=BUCKET_NAME,
58+
target_filename=FILENAME,
59+
task_id="text_to_speech_synthesize_task",
60+
)
61+
# [START howto_operator_speech_to_text_recognize]
62+
speech_to_text_recognize_task2 = CloudSpeechToTextRecognizeSpeechOperator(
63+
config=CONFIG,
64+
audio=AUDIO,
65+
task_id="speech_to_text_recognize_task"
66+
)
67+
# [END howto_operator_speech_to_text_recognize]
68+
69+
text_to_speech_synthesize_task >> speech_to_text_recognize_task2

‎airflow/providers/google/cloud/example_dags/example_text_to_speech.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
import os
20+
21+
from airflow import models
22+
from airflow.providers.google.cloud.operators.text_to_speech import CloudTextToSpeechSynthesizeOperator
23+
from airflow.utils import dates
24+
25+
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
26+
BUCKET_NAME = os.environ.get("GCP_TEXT_TO_SPEECH_BUCKET", "gcp-text-to-speech-test-bucket")
27+
28+
# [START howto_operator_text_to_speech_gcp_filename]
29+
FILENAME = "gcp-speech-test-file"
30+
# [END howto_operator_text_to_speech_gcp_filename]
31+
32+
# [START howto_operator_text_to_speech_api_arguments]
33+
INPUT = {"text": "Sample text for demo purposes"}
34+
VOICE = {"language_code": "en-US", "ssml_gender": "FEMALE"}
35+
AUDIO_CONFIG = {"audio_encoding": "LINEAR16"}
36+
# [END howto_operator_text_to_speech_api_arguments]
37+
38+
default_args = {"start_date": dates.days_ago(1)}
39+
40+
with models.DAG(
41+
"example_gcp_text_to_speech",
42+
default_args=default_args,
43+
schedule_interval=None, # Override to match your needs
44+
tags=['example'],
45+
) as dag:
46+
47+
# [START howto_operator_text_to_speech_synthesize]
48+
text_to_speech_synthesize_task = CloudTextToSpeechSynthesizeOperator(
49+
project_id=GCP_PROJECT_ID,
50+
input_data=INPUT,
51+
voice=VOICE,
52+
audio_config=AUDIO_CONFIG,
53+
target_bucket_name=BUCKET_NAME,
54+
target_filename=FILENAME,
55+
task_id="text_to_speech_synthesize_task",
56+
)
57+
text_to_speech_synthesize_task2 = CloudTextToSpeechSynthesizeOperator(
58+
input_data=INPUT,
59+
voice=VOICE,
60+
audio_config=AUDIO_CONFIG,
61+
target_bucket_name=BUCKET_NAME,
62+
target_filename=FILENAME,
63+
task_id="text_to_speech_synthesize_task2",
64+
)
65+
# [END howto_operator_text_to_speech_synthesize]
66+
67+
text_to_speech_synthesize_task >> text_to_speech_synthesize_task2

airflow/providers/google/cloud/example_dags/example_speech.py renamed to airflow/providers/google/cloud/example_dags/example_translate_speech.py

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -16,41 +16,29 @@
1616
# specific language governing permissions and limitations
1717
# under the License.
1818

19-
"""
20-
Example Airflow DAG that runs speech synthesizing and stores output in Google Cloud Storage
21-
22-
This DAG relies on the following OS environment variables
23-
* GCP_PROJECT_ID - Google Cloud Platform project for the Cloud SQL instance.
24-
* GCP_SPEECH_TEST_BUCKET - Name of the bucket in which the output file should be stored.
25-
"""
26-
2719
import os
2820

2921
from airflow import models
30-
from airflow.providers.google.cloud.operators.speech_to_text import CloudSpeechToTextRecognizeSpeechOperator
3122
from airflow.providers.google.cloud.operators.text_to_speech import CloudTextToSpeechSynthesizeOperator
3223
from airflow.providers.google.cloud.operators.translate_speech import CloudTranslateSpeechOperator
3324
from airflow.utils import dates
3425

3526
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
36-
BUCKET_NAME = os.environ.get("GCP_SPEECH_TEST_BUCKET", "gcp-speech-test-bucket")
27+
BUCKET_NAME = os.environ.get("GCP_TRANSLATE_SPEECH_TEST_BUCKET", "gcp-translate-speech-test-bucket")
3728

38-
# [START howto_operator_text_to_speech_gcp_filename]
29+
# [START howto_operator_translate_speech_gcp_filename]
3930
FILENAME = "gcp-speech-test-file"
40-
# [END howto_operator_text_to_speech_gcp_filename]
31+
# [END howto_operator_translate_speech_gcp_filename]
4132

4233
# [START howto_operator_text_to_speech_api_arguments]
4334
INPUT = {"text": "Sample text for demo purposes"}
4435
VOICE = {"language_code": "en-US", "ssml_gender": "FEMALE"}
4536
AUDIO_CONFIG = {"audio_encoding": "LINEAR16"}
4637
# [END howto_operator_text_to_speech_api_arguments]
4738

48-
# [START howto_operator_speech_to_text_api_arguments]
39+
# [START howto_operator_translate_speech_arguments]
4940
CONFIG = {"encoding": "LINEAR16", "language_code": "en_US"}
5041
AUDIO = {"uri": "gs://{bucket}/{object}".format(bucket=BUCKET_NAME, object=FILENAME)}
51-
# [END howto_operator_speech_to_text_api_arguments]
52-
53-
# [START howto_operator_translate_speech_arguments]
5442
TARGET_LANGUAGE = 'pl'
5543
FORMAT = 'text'
5644
MODEL = 'base'
@@ -60,13 +48,11 @@
6048
default_args = {"start_date": dates.days_ago(1)}
6149

6250
with models.DAG(
63-
"example_gcp_speech",
51+
"example_gcp_translate_speech",
6452
default_args=default_args,
6553
schedule_interval=None, # Override to match your needs
6654
tags=['example'],
6755
) as dag:
68-
69-
# [START howto_operator_text_to_speech_synthesize]
7056
text_to_speech_synthesize_task = CloudTextToSpeechSynthesizeOperator(
7157
project_id=GCP_PROJECT_ID,
7258
input_data=INPUT,
@@ -76,16 +62,6 @@
7662
target_filename=FILENAME,
7763
task_id="text_to_speech_synthesize_task",
7864
)
79-
# [END howto_operator_text_to_speech_synthesize]
80-
81-
# [START howto_operator_speech_to_text_recognize]
82-
speech_to_text_recognize_task = CloudSpeechToTextRecognizeSpeechOperator(
83-
project_id=GCP_PROJECT_ID, config=CONFIG, audio=AUDIO, task_id="speech_to_text_recognize_task"
84-
)
85-
# [END howto_operator_speech_to_text_recognize]
86-
87-
text_to_speech_synthesize_task >> speech_to_text_recognize_task
88-
8965
# [START howto_operator_translate_speech]
9066
translate_speech_task = CloudTranslateSpeechOperator(
9167
project_id=GCP_PROJECT_ID,
@@ -97,6 +73,14 @@
9773
model=MODEL,
9874
task_id='translate_speech_task'
9975
)
76+
translate_speech_task2 = CloudTranslateSpeechOperator(
77+
audio=AUDIO,
78+
config=CONFIG,
79+
target_language=TARGET_LANGUAGE,
80+
format_=FORMAT,
81+
source_language=SOURCE_LANGUAGE,
82+
model=MODEL,
83+
task_id='translate_speech_task2'
84+
)
10085
# [END howto_operator_translate_speech]
101-
102-
text_to_speech_synthesize_task >> translate_speech_task
86+
text_to_speech_synthesize_task >> translate_speech_task >> translate_speech_task2

‎docs/howto/operator/gcp/speech_to_text.rst‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
.. Licensed to the Apache Software Foundation (ASF) under one
2+
or more contributor license agreements. See the NOTICE file
3+
distributed with this work for additional information
4+
regarding copyright ownership. The ASF licenses this file
5+
to you under the Apache License, Version 2.0 (the
6+
"License"); you may not use this file except in compliance
7+
with the License. You may obtain a copy of the License at
8+
9+
.. http://www.apache.org/licenses/LICENSE-2.0
10+
11+
.. Unless required by applicable law or agreed to in writing,
12+
software distributed under the License is distributed on an
13+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
KIND, either express or implied. See the License for the
15+
specific language governing permissions and limitations
16+
under the License.
17+
18+
19+
Google Cloud Speech to Text Operators
20+
=====================================
21+
22+
Prerequisite Tasks
23+
------------------
24+
25+
.. include:: _partials/prerequisite_tasks.rst
26+
27+
.. _howto/operator:CloudSpeechToTextRecognizeSpeechOperator:
28+
29+
CloudSpeechToTextRecognizeSpeechOperator
30+
----------------------------------------
31+
32+
Recognizes speech in audio input and returns text.
33+
34+
For parameter definition, take a look at
35+
:class:`~airflow.providers.google.cloud.operators.speech_to_text.CloudSpeechToTextRecognizeSpeechOperator`
36+
37+
Arguments
38+
"""""""""
39+
40+
config and audio arguments need to be dicts or objects of corresponding classes from
41+
google.cloud.speech_v1.types module
42+
43+
for more information, see: https://googleapis.github.io/google-cloud-python/latest/speech/gapic/v1/api.html#google.cloud.speech_v1.SpeechClient.recognize
44+
45+
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
46+
:language: python
47+
:start-after: [START howto_operator_text_to_speech_api_arguments]
48+
:end-before: [END howto_operator_text_to_speech_api_arguments]
49+
50+
filename is a simple string argument:
51+
52+
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
53+
:language: python
54+
:start-after: [START howto_operator_speech_to_text_api_arguments]
55+
:end-before: [END howto_operator_speech_to_text_api_arguments]
56+
57+
Using the operator
58+
""""""""""""""""""
59+
60+
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
61+
:language: python
62+
:dedent: 4
63+
:start-after: [START howto_operator_speech_to_text_recognize]
64+
:end-before: [END howto_operator_speech_to_text_recognize]
65+
66+
Templating
67+
""""""""""
68+
69+
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/speech_to_text.py
70+
:language: python
71+
:dedent: 4
72+
:start-after: [START gcp_speech_to_text_synthesize_template_fields]
73+
:end-before: [END gcp_speech_to_text_synthesize_template_fields]
74+
75+
Reference
76+
---------
77+
78+
For further information, look at:
79+
80+
* `Client Library Documentation <https://googleapis.github.io/google-cloud-python/latest/speech/>`__
81+
* `Product Documentation <https://cloud.google.com/speech/>`__

0 commit comments

Comments
 (0)