almost final

README.md

@@ -41,4 +41,4 @@ Since ETLs are built as Airflow tasks within a DAG, they may be triggered automa
 ---
-Screenshots to each steps are attached to the bachelor thesis
+Screenshots to each steps are attached to the bachelor thesis.

airflow/airflow-webserver.pid

+26389

airflow/dags/ezop/ezop_config.py

@@ -6,10 +6,10 @@ EZOP_SRC_DB = None
 EZOP_SRC_SCHEMA = 'VVVS_REST'
 EZOP_STG_CONN_ID = EZOP_TG_CONN_ID = 'dv5'
-EZOP_STG_DB = 'dwh3_test_stage'
+EZOP_STG_DB = 'dwh3_stage_kika'
 EZOP_STG_SCHEMA = 'ps_ezop'
-EZOP_TG_DB = 'dwh3_test_target'
+EZOP_TG_DB = 'dwh3_target_kika'
 EZOP_TG_SCHEMA = 'dwh'
 #===============================================================================

airflow/dags/grades/grades_config.py

@@ -6,10 +6,10 @@ GRADES_SRC_DB = 'classification'
 GRADES_SRC_SCHEMA = 'main'
 GRADES_STG_CONN_ID = GRADES_TG_CONN_ID = 'dv5'
-GRADES_STG_DB = 'dwh3_test_stage'
+GRADES_STG_DB = 'dwh3_stage_kika'
 GRADES_STG_SCHEMA = 'ps_grades'
-GRADES_TG_DB = 'dwh3_test_target'
+GRADES_TG_DB = 'dwh3_target_kika'
 GRADES_TG_SCHEMA = 'dwh'
 #===============================================================================

airflow/dags/kos/kos_config.py

@@ -6,10 +6,10 @@ KOS_SRC_DB = None
 KOS_SRC_SCHEMA = 'ST'
 KOS_STG_CONN_ID = KOS_TG_CONN_ID = 'dv5'
-KOS_STG_DB = 'dwh3_test_stage'
+KOS_STG_DB = 'dwh3_stage_kika'
 KOS_STG_SCHEMA = 'ps_kos'
-KOS_TG_DB = 'dwh3_test_target'
+KOS_TG_DB = 'dwh3_target_kika'
 KOS_TG_SCHEMA = 'dwh'
 #===============================================================================

airflow/dags/usermap/usermap_config.py

@@ -8,10 +8,10 @@ USERMAP_SRC_DB = None
 USERMAP_SRC_SCHEMA = 'EXT_DS'
 USERMAP_STG_CONN_ID = USERMAP_TG_CONN_ID = 'dv5'
-USERMAP_STG_DB = 'dwh3_test_stage'
+USERMAP_STG_DB = 'dwh3_stage_kika'
 USERMAP_STG_SCHEMA = 'ps_usermap'
-USERMAP_TG_DB = 'dwh3_test_target'
+USERMAP_TG_DB = 'dwh3_target_kika'
 USERMAP_TG_SCHEMA = 'dwh'
 #===============================================================================

airflow/logs/scheduler/latest

-/home/zolockri/Documents/FIT/BP/airflow_workspace/airflow/logs/scheduler/2023-05-05
\ No newline at end of file
+/home/zolockri/Documents/FIT/BP/airflow_workspace/airflow/logs/scheduler/2023-05-06
\ No newline at end of file

airflow/plugins/custom_operators/helpers/process_data_helpers.py

-from helpers.stringiteratorio import StringIteratorIO
-
-def get_data_iterator(data):
-    data = [tuple(row) for row in data.itertuples(index=False)]
-    return StringIteratorIO((
-            "\x1f".join(map(str, row)) + '\n')
-            for row in data
-        )