From c35a491ab9b70b024fc4a561daeb7b96d8eecd30 Mon Sep 17 00:00:00 2001
From: Kristina Zolochevskaia <zolockri@fit.cvut.cz>
Date: Sat, 6 May 2023 18:36:23 +0200
Subject: [PATCH] almost final

---
 README.md                                                 | 2 +-
 airflow/airflow-webserver.pid                             | 1 +
 airflow/dags/ezop/ezop_config.py                          | 4 ++--
 airflow/dags/grades/grades_config.py                      | 4 ++--
 airflow/dags/kos/kos_config.py                            | 4 ++--
 airflow/dags/usermap/usermap_config.py                    | 4 ++--
 airflow/logs/scheduler/latest                             | 2 +-
 .../custom_operators/helpers/process_data_helpers.py      | 8 --------
 8 files changed, 11 insertions(+), 18 deletions(-)
 create mode 100644 airflow/airflow-webserver.pid
 delete mode 100644 airflow/plugins/custom_operators/helpers/process_data_helpers.py

diff --git a/README.md b/README.md
index ad138f4..425746d 100644
--- a/README.md
+++ b/README.md
@@ -41,4 +41,4 @@ Since ETLs are built as Airflow tasks within a DAG, they may be triggered automa
 
 ---
 
-Screenshots to each steps are attached to the bachelor thesis
+Screenshots to each steps are attached to the bachelor thesis.
diff --git a/airflow/airflow-webserver.pid b/airflow/airflow-webserver.pid
new file mode 100644
index 0000000..bb68521
--- /dev/null
+++ b/airflow/airflow-webserver.pid
@@ -0,0 +1 @@
+26389
diff --git a/airflow/dags/ezop/ezop_config.py b/airflow/dags/ezop/ezop_config.py
index 5051b85..443199e 100644
--- a/airflow/dags/ezop/ezop_config.py
+++ b/airflow/dags/ezop/ezop_config.py
@@ -6,10 +6,10 @@ EZOP_SRC_DB = None
 EZOP_SRC_SCHEMA = 'VVVS_REST'
 
 EZOP_STG_CONN_ID = EZOP_TG_CONN_ID = 'dv5'
-EZOP_STG_DB = 'dwh3_test_stage'
+EZOP_STG_DB = 'dwh3_stage_kika'
 EZOP_STG_SCHEMA = 'ps_ezop'
 
-EZOP_TG_DB = 'dwh3_test_target'
+EZOP_TG_DB = 'dwh3_target_kika'
 EZOP_TG_SCHEMA = 'dwh'
 
 #===============================================================================
diff --git a/airflow/dags/grades/grades_config.py b/airflow/dags/grades/grades_config.py
index 95a7c62..f6f5884 100644
--- a/airflow/dags/grades/grades_config.py
+++ b/airflow/dags/grades/grades_config.py
@@ -6,10 +6,10 @@ GRADES_SRC_DB = 'classification'
 GRADES_SRC_SCHEMA = 'main'
 
 GRADES_STG_CONN_ID = GRADES_TG_CONN_ID = 'dv5'
-GRADES_STG_DB = 'dwh3_test_stage'
+GRADES_STG_DB = 'dwh3_stage_kika'
 GRADES_STG_SCHEMA = 'ps_grades'
 
-GRADES_TG_DB = 'dwh3_test_target'
+GRADES_TG_DB = 'dwh3_target_kika'
 GRADES_TG_SCHEMA = 'dwh'
 
 #===============================================================================
diff --git a/airflow/dags/kos/kos_config.py b/airflow/dags/kos/kos_config.py
index 2f6440d..1512d70 100644
--- a/airflow/dags/kos/kos_config.py
+++ b/airflow/dags/kos/kos_config.py
@@ -6,10 +6,10 @@ KOS_SRC_DB = None
 KOS_SRC_SCHEMA = 'ST'
 
 KOS_STG_CONN_ID = KOS_TG_CONN_ID = 'dv5'
-KOS_STG_DB = 'dwh3_test_stage'
+KOS_STG_DB = 'dwh3_stage_kika'
 KOS_STG_SCHEMA = 'ps_kos'
 
-KOS_TG_DB = 'dwh3_test_target'
+KOS_TG_DB = 'dwh3_target_kika'
 KOS_TG_SCHEMA = 'dwh'
 
 #===============================================================================
diff --git a/airflow/dags/usermap/usermap_config.py b/airflow/dags/usermap/usermap_config.py
index 4751bb8..27e10db 100644
--- a/airflow/dags/usermap/usermap_config.py
+++ b/airflow/dags/usermap/usermap_config.py
@@ -8,10 +8,10 @@ USERMAP_SRC_DB = None
 USERMAP_SRC_SCHEMA = 'EXT_DS'
 
 USERMAP_STG_CONN_ID = USERMAP_TG_CONN_ID = 'dv5'
-USERMAP_STG_DB = 'dwh3_test_stage'
+USERMAP_STG_DB = 'dwh3_stage_kika'
 USERMAP_STG_SCHEMA = 'ps_usermap'
 
-USERMAP_TG_DB = 'dwh3_test_target'
+USERMAP_TG_DB = 'dwh3_target_kika'
 USERMAP_TG_SCHEMA = 'dwh'
 
 #===============================================================================
diff --git a/airflow/logs/scheduler/latest b/airflow/logs/scheduler/latest
index e82c5cc..888a2d8 120000
--- a/airflow/logs/scheduler/latest
+++ b/airflow/logs/scheduler/latest
@@ -1 +1 @@
-/home/zolockri/Documents/FIT/BP/airflow_workspace/airflow/logs/scheduler/2023-05-05
\ No newline at end of file
+/home/zolockri/Documents/FIT/BP/airflow_workspace/airflow/logs/scheduler/2023-05-06
\ No newline at end of file
diff --git a/airflow/plugins/custom_operators/helpers/process_data_helpers.py b/airflow/plugins/custom_operators/helpers/process_data_helpers.py
deleted file mode 100644
index a2b7576..0000000
--- a/airflow/plugins/custom_operators/helpers/process_data_helpers.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from helpers.stringiteratorio import StringIteratorIO
-
-def get_data_iterator(data):
-    data = [tuple(row) for row in data.itertuples(index=False)]
-    return StringIteratorIO((
-            "\x1f".join(map(str, row)) + '\n')
-            for row in data
-        )
-- 
GitLab