-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrapper.py
36 lines (27 loc) · 871 Bytes
/
scrapper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
from datetime import datetime
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.operators.bash_operator import BashOperator
from airflow.models import Variable
import jovem_nerd
default_args = {
"owner": "airflow",
"start_date": datetime(2023, 10, 26),
}
dag = DAG(
"jovem_nerd_dag",
default_args=default_args,
description="DAG que criar um dataset com todos os episódios dos podcasts do jovem nerd",
schedule_interval=None,
)
get_jovem_nerd_operator = PythonOperator(
task_id="get_jovem_nerd_data",
python_callable=jovem_nerd.main,
dag=dag,
)
send_datatase_kaggle = BashOperator(
task_id="send_datatase_kaggle",
bash_command="kaggle datasets version -p /home/var/data/ -m 'Automatic Update via Airflow'",
dag=dag,
)
get_jovem_nerd_operator >> send_datatase_kaggle