#1406 - fix all warehouse connections

Derive PostgreSQL connect string for data warehouse DB from Django connections, then create a single engine for all queries that will use that DB.

Author: lsloan

dashboard/cron.py

@@ -38,6 +38,11 @@
                                host=db_host,
                                port=db_port))
 
+data_warehouse_engine = create_engine(
+    'postgresql://{user}:{password}@{host}:{port}/{database}'
+    .format(**(conns['DATA_WAREHOUSE'].get_connection_params())))
+
+
 # Set up queries array from configuration file
 CRON_QUERY_FILE = settings.CRON_QUERY_FILE
 logger.info(CRON_QUERY_FILE)
@@ -59,7 +64,7 @@ def split_list(a_list: list, size: int = 20):
 
 
 def util_function(data_warehouse_course_id, sql_string, mysql_table, table_identifier=None, param_object=None):
-    df = pd.read_sql(sql_string, conns['DATA_WAREHOUSE'], params=param_object)
+    df = pd.read_sql(sql_string, data_warehouse_engine, params=param_object)
     logger.debug(df)
 
     # Sql returns boolean value so grouping course info along with it so that this could be stored in the DB table.
@@ -134,7 +139,8 @@ def __init__(self) -> None:
         super().__init__()
         self.valid_locked_course_ids: List[int]
 
-    # verify whether course ids are valid
+
+# verify whether course ids are valid
     def verify_course_ids(self):
         # whether all course ids are valid ids
         invalid_course_id_list = []
@@ -149,10 +155,7 @@ def verify_course_ids(self):
             course_sql = queries['course'].format(course_id=course_id)
             logger.debug(course_sql)
 
-            connect_string = \
-                'postgresql://{user}:{password}@{host}:{port}/{database}' \
-                    .format(**(conns['DATA_WAREHOUSE'].get_connection_params()))
-            course_df = pd.read_sql(course_sql, connect_string)
+            course_df = pd.read_sql(course_sql, data_warehouse_engine)
             logger.debug(course_df)
 
             # error out when course id is invalid, otherwise add DataFrame to list
@@ -230,7 +233,8 @@ def update_canvas_resource(self):
         course_ids = list(map(str, self.valid_locked_course_ids))
         file_sql = queries['resource']
         logger.debug(file_sql)
-        df_attach = pd.read_sql(file_sql, conns['DATA_WAREHOUSE'], params={'course_ids': tuple(course_ids)})
+        df_attach = pd.read_sql(file_sql, data_warehouse_engine,
+                                params={'course_ids': tuple(course_ids)})
         logger.debug(df_attach)
         # Update these back again based on the dataframe
         # Remove any rows where file_state is not available!
@@ -553,7 +557,8 @@ def update_term(self) -> str:
 
         term_sql: str = queries['term']
         logger.debug(term_sql)
-        warehouse_term_df: pd.DataFrame = pd.read_sql(term_sql, conns['DATA_WAREHOUSE'])
+        warehouse_term_df: pd.DataFrame = pd.read_sql(term_sql,
+                                                      data_warehouse_engine)
 
         existing_terms_ids: List[int] = [term.id for term in list(AcademicTerms.objects.all())]
         new_term_ids: List[int] = [int(id) for id in warehouse_term_df['id'].to_list() if id not in existing_terms_ids]