Merge pull request #153 from TeachMeTW/Refactor-User-Stats

 Integrate New User Statistics into Profile Retrieval

Author: shankari

utils/constants.py

@@ -35,7 +35,7 @@
     "inferred_section_summary",
 ]
 
-valid_uuids_columns = [
+VALID_UUIDS_COLS = [
     'user_token',
     'user_id',
     'update_ts',

utils/db_utils.py

@@ -437,57 +437,35 @@ def add_user_stats(user_data, batch_size=5):
         def process_user(user):
             with ect.Timer() as process_user_timer:
                 user_uuid = UUID(user['user_id'])
-                
-                # Fetch aggregated data for all users once and cache it
-                ts_aggregate = esta.TimeSeries.get_aggregate_time_series()
-    
-                # Fetch data for the user, cached for repeated queries
                 profile_data = edb.get_profile_db().find_one({'user_id': user_uuid})
+                # Fetch data for the user, cached for repeated queries
+                logging.info(f'keyspr: {profile_data}')
+                if not profile_data:
+                    profile_data = {}
+                # Assign existing profile attributes to the user dictionary
+                user['platform'] = profile_data.get('curr_platform')
+                user['manufacturer'] = profile_data.get('manufacturer')
+                user['app_version'] = profile_data.get('client_app_version')
+                user['os_version'] = profile_data.get('client_os_version')
+                user['phone_lang'] = profile_data.get('phone_lang')
 
-                total_trips = ts_aggregate.find_entries_count(
-                    key_list=["analysis/confirmed_trip"],
-                    extra_query_list=[{'user_id': user_uuid}]
-                )
-                labeled_trips = ts_aggregate.find_entries_count(
-                    key_list=["analysis/confirmed_trip"],
-                    extra_query_list=[{'user_id': user_uuid}, {'data.user_input': {'$ne': {}}}]
-                )
+                # Assign newly stored statistics to the user dictionary
+                user['total_trips'] = profile_data.get('total_trips')
+                user['labeled_trips'] = profile_data.get('labeled_trips')
 
-                user['total_trips'] = total_trips
-                user['labeled_trips'] = labeled_trips
-    
-                if profile_data:
-                    user['platform'] = profile_data.get('curr_platform')
-                    user['manufacturer'] = profile_data.get('manufacturer')
-                    user['app_version'] = profile_data.get('client_app_version')
-                    user['os_version'] = profile_data.get('client_os_version')
-                    user['phone_lang'] = profile_data.get('phone_lang')
-    
-                if total_trips > 0:
-                    ts = esta.TimeSeries.get_time_series(user_uuid)
-                    first_trip_ts = ts.get_first_value_for_field(
-                        key='analysis/confirmed_trip',
-                        field='data.end_ts',
-                        sort_order=pymongo.ASCENDING
-                    )
-                    if first_trip_ts != -1:
-                        user['first_trip'] = arrow.get(first_trip_ts).format(time_format)
-    
-                    last_trip_ts = ts.get_first_value_for_field(
-                        key='analysis/confirmed_trip',
-                        field='data.end_ts',
-                        sort_order=pymongo.DESCENDING
-                    )
-                    if last_trip_ts != -1:
-                        user['last_trip'] = arrow.get(last_trip_ts).format(time_format)
-    
-                    last_call_ts = ts.get_first_value_for_field(
-                        key='stats/server_api_time',
-                        field='data.ts',
-                        sort_order=pymongo.DESCENDING
-                    )
-                    if last_call_ts != -1:
-                        user['last_call'] = arrow.get(last_call_ts).format(time_format)
+                # Retrieve and assign pipeline range
+                pipeline_range = profile_data.get('pipeline_range', {})
+                start_ts = pipeline_range.get('start_ts')
+                end_ts = pipeline_range.get('end_ts')
+                if start_ts:
+                    user['first_trip'] = arrow.get(start_ts).format(time_format)
+                if end_ts:
+                    user['last_trip'] = arrow.get(end_ts).format(time_format)
+                
+                # Retrieve and assign last API call timestamp
+                last_call_ts = profile_data.get('last_call_ts')
+                if last_call_ts:
+                    user['last_call'] = arrow.get(last_call_ts).format('YYYY-MM-DD')
 
             esdsq.store_dashboard_time(
                 "admin/db_utils/add_user_stats/process_user",

utils/permissions.py

@@ -93,7 +93,7 @@ def get_allowed_trip_columns():
 
 
 def get_uuids_columns():
-    columns = set(constants.valid_uuids_columns)
+    columns = set(constants.VALID_UUIDS_COLS)
     for column in permissions.get("data_uuids_columns_exclude", []):
         columns.discard(column)
     return columns