Merge pull request #34 from shankari/gamification

Add server and result level stats storage

Author: shankari

CFC_WebApp/api/cfc_webapp.py

@@ -7,6 +7,7 @@
 import os
 import logging
 from datetime import datetime
+import time
 # So that we can set the socket timeout
 import socket
 # For decoding tokens using the google decode URL
@@ -317,12 +318,18 @@ def movesCallback():
 @app.hook('before_request')
 def before_request():
   print("START %s %s %s" % (datetime.now(), request.method, request.path))
+  request.params.start_ts = time.time()
   logging.debug("START %s %s" % (request.method, request.path))
 
 @app.hook('after_request')
 def after_request():
-  print("END %s %s %s" % (datetime.now(), request.method, request.path))
-  logging.debug("END %s %s" % (request.method, request.path))
+  msTimeNow = time.time()
+  duration = msTimeNow - request.params.start_ts
+  print("END %s %s %s %s %s " % (datetime.now(), request.method, request.path, request.params.user_uuid, duration))
+  logging.debug("END %s %s %s %s " % (request.method, request.path, request.params.user_uuid, duration))
+  # Keep track of the time and duration for each call
+  stats.storeServerEntry(request.params.user_uuid, "%s %s" % (request.method, request.path),
+        msTimeNow, duration)
 
 # Auth helpers BEGIN
 # This should only be used by createUserProfile since we may not have a UUID
@@ -347,6 +354,7 @@ def getUUIDFromToken(token):
     return user_uuid
 
 def getUUID(request):
+  retUUID = None
   if skipAuth:
     from uuid import UUID
     from get_database import get_uuid_db
@@ -355,11 +363,13 @@ def getUUID(request):
     else:
       # TODO: Figure out what we really want to do here
       user_uuid = UUID('{3a307244-ecf1-3e6e-a9a7-3aaf101b40fa}')
+    retUUID = user_uuid
     logging.debug("skipAuth = %s, returning fake UUID %s" % (skipAuth, user_uuid))
-    return user_uuid
   else:
     userToken = request.json['user']
-    return getUUIDFromToken(userToken)
+    retUUID = getUUIDFromToken(userToken)
+  request.params.user_uuid = retUUID
+  return retUUID
 # Auth helpers END
 
 # We have see the sockets hang in practice. Let's set the socket timeout = 1

CFC_WebApp/clients/choice/choice.py

@@ -1,18 +1,20 @@
 import logging
 from get_database import get_section_db
-from main import carbon, common
+from main import carbon, common, stats
 from datetime import datetime, time, timedelta
 from dao.user import User
 import math
 import json
 from uuid import UUID
+import time
 from clients.gamified import gamified
 from clients.default import default
 
 # TODO: Consider subclassing to provide client specific user functions
 def setCurrView(uuid, newView):
   user = User.fromUUID(uuid)
   user.setClientSpecificProfileFields({'curr_view': newView})
+  stats.storeResultEntry(uuid, stats.STAT_VIEW_CHOICE, time.time(), newView)
 
 def getCurrView(uuid):
   user = User.fromUUID(uuid)

CFC_WebApp/clients/default/default.py

@@ -1,6 +1,7 @@
 import logging
-from main import carbon
+from main import carbon, stats
 from dao.user import User
+import time
 import json
 
 # BEGIN: Code to get and set client specific fields in the profile (currentScore and previousScore)
@@ -12,7 +13,6 @@ def getCarbonFootprint(user):
 
 def setCarbonFootprint(user, newFootprint):
     user.setClientSpecificProfileFields({'carbon_footprint': newFootprint})
-
 # END: Code to get and set client specific fields in the profile (currentScore and previousScore)
 
 def getResult(user_uuid):
@@ -47,6 +47,9 @@ def getResult(user_uuid):
   # logging.debug(renderedTemplate)
   return renderedTemplate
 
+def getCategorySum(carbonFootprintMap):
+    return sum(carbonFootprintMap.values())
+
 def runBackgroundTasks(user_uuid):
   user = User.fromUUID(user_uuid)
   # carbon compare results is a tuple. Tuples are converted to arrays
@@ -57,3 +60,20 @@ def runBackgroundTasks(user_uuid):
   carbonCompareResults = carbon.getFootprintCompare(user_uuid)
   setCarbonFootprint(user, carbonCompareResults)
 
+  (myModeShareCount, avgModeShareCount,
+     myModeShareDistance, avgModeShareDistance,
+     myModeCarbonFootprint, avgModeCarbonFootprint,
+     myModeCarbonFootprintNoLongMotorized, avgModeCarbonFootprintNoLongMotorized, # ignored
+     myOptimalCarbonFootprint, avgOptimalCarbonFootprint,
+     myOptimalCarbonFootprintNoLongMotorized, avgOptimalCarbonFootprintNoLongMotorized) = carbonCompareResults
+  # We only compute server stats in the background, because including them in
+  # the set call means that they may be invoked when the user makes a call and
+  # the cached value is None, which would potentially slow down user response time
+  msNow = time.time()
+  stats.storeResultEntry(user_uuid, stats.STAT_MY_CARBON_FOOTPRINT, msNow, getCategorySum(myModeCarbonFootprint))
+  stats.storeResultEntry(user_uuid, stats.STAT_MY_CARBON_FOOTPRINT_NO_AIR, msNow, getCategorySum(myModeCarbonFootprintNoLongMotorized))
+  stats.storeResultEntry(user_uuid, stats.STAT_MY_OPTIMAL_FOOTPRINT, msNow, getCategorySum(myOptimalCarbonFootprint))
+  stats.storeResultEntry(user_uuid, stats.STAT_MY_OPTIMAL_FOOTPRINT_NO_AIR, msNow, getCategorySum(myOptimalCarbonFootprintNoLongMotorized))
+  stats.storeResultEntry(user_uuid, stats.STAT_MY_ALLDRIVE_FOOTPRINT, msNow, getCategorySum(myModeShareDistance) * (278.0/(1609 * 1000)))
+  stats.storeResultEntry(user_uuid, stats.STAT_MEAN_FOOTPRINT, msNow, getCategorySum(avgModeCarbonFootprint))
+  stats.storeResultEntry(user_uuid, stats.STAT_MEAN_FOOTPRINT_NO_AIR, msNow, getCategorySum(avgModeCarbonFootprintNoLongMotorized))

CFC_WebApp/clients/gamified/gamified.py

@@ -1,7 +1,9 @@
 import logging
 from get_database import get_section_db
-from main import carbon, common
-from datetime import datetime, time, timedelta
+from main import carbon, common, stats
+from datetime import datetime, timedelta
+from datetime import time as dttime
+import time
 from dao.user import User
 import math
 
@@ -85,14 +87,15 @@ def getScore(user_uuid, start, end):
 def updateScore(user_uuid):
     today = datetime.now().date()
     yesterday = today - timedelta(days = 1)
-    yesterdayStart = datetime.combine(yesterday, time.min)
-    todayStart = datetime.combine(today, time.min)
+    yesterdayStart = datetime.combine(yesterday, dttime.min)
+    todayStart = datetime.combine(today, dttime.min)
 
     user = User.fromUUID(user_uuid)
     (discardedScore, prevScore) = getStoredScore(user)
     newScore = prevScore + getScore(user_uuid, yesterdayStart, todayStart)
     if newScore < 0:
         newScore = 0
+    stats.storeResultEntry(user_uuid, stats.STAT_GAME_SCORE, time.time(), newScore)
     setScores(user, prevScore, newScore)
 
 def getLevel(score):

CFC_WebApp/get_database.py

@@ -61,6 +61,16 @@ def get_client_stats_db():
     ClientStats = current_db.Stage_client_stats
     return ClientStats
 
+def get_server_stats_db():
+    current_db=MongoClient().Stage_database
+    ServerStats = current_db.Stage_server_stats
+    return ServerStats
+
+def get_result_stats_db():
+    current_db=MongoClient().Stage_database
+    ResultStats = current_db.Stage_result_stats
+    return ResultStats
+
 def get_db():
     current_db=MongoClient('localhost').Stage_database
     return current_db

CFC_WebApp/main/get_database.py

@@ -62,6 +62,16 @@ def get_client_stats_db():
     ClientStats = current_db.Stage_client_stats
     return ClientStats
 
+def get_server_stats_db():
+    current_db=MongoClient().Stage_database
+    ServerStats = current_db.Stage_server_stats
+    return ServerStats
+
+def get_result_stats_db():
+    current_db=MongoClient().Stage_database
+    ResultStats = current_db.Stage_result_stats
+    return ResultStats
+
 def get_db():
     current_db=MongoClient('localhost').Stage_database
     return current_db

CFC_WebApp/main/stats.py

@@ -1,7 +1,22 @@
 import logging
 import time
-from get_database import get_client_stats_db
+from get_database import get_client_stats_db, get_server_stats_db, get_result_stats_db
 
+STAT_TRIP_MGR_PCT_SHOWN = "tripManager.pctShown"
+STAT_MY_CARBON_FOOTPRINT = "footprint.my_carbon"
+STAT_MY_CARBON_FOOTPRINT_NO_AIR = "footprint.my_carbon.no_air"
+STAT_MY_OPTIMAL_FOOTPRINT = "footprint.optimal"
+STAT_MY_OPTIMAL_FOOTPRINT_NO_AIR = "footprint.optimal.no_air"
+STAT_MY_ALLDRIVE_FOOTPRINT = "footprint.alldrive"
+STAT_MEAN_FOOTPRINT = "footprint.mean"
+STAT_MEAN_FOOTPRINT_NO_AIR = "footprint.mean.no_air"
+STAT_GAME_SCORE = "game.score"
+STAT_VIEW_CHOICE = "view.choice"
+
+# Store client measurements (currently into the database, but maybe in a log
+# file in the future). The format of the stats received from the client is very
+# similar to the input to SMAP, to make it easier to store them in a SMAP
+# database in the future
 def setClientMeasurements(user, reportedVals):
   logging.info("Received %d client keys and %d client readings for user %s" % (len(reportedVals['Readings']),
     getClientMeasurementCount(reportedVals['Readings']), user))
@@ -12,10 +27,29 @@ def setClientMeasurements(user, reportedVals):
   for key in stats:
     values = stats[key]
     for value in values:
-      currEntry = createEntry(user, key, value[0], value[1])
-      # Add the os and app versions from the metadata dict
-      currEntry.update(metadata)
-      get_client_stats_db().insert(currEntry)
+      storeClientEntry(user, key, value[0], value[1], metadata)
+
+def storeClientEntry(user, key, ts, reading, metadata):
+  logging.debug("storing client entry for user %s, key %s at timestamp %s" % (user, key, ts))
+  currEntry = createEntry(user, key, ts, reading)
+  # Add the os and app versions from the metadata dict
+  currEntry.update(metadata)
+  get_client_stats_db().insert(currEntry)
+
+# server measurements will call this directly since there's not much point in
+# batching in a different location and then making a call here since it runs on
+# the same server. Might change if we move engagement stats to a different server
+# Note also that there is no server metadata since we currently have no
+# versioning on the server. Should probably add some soon
+def storeServerEntry(user, key, ts, reading):
+  logging.debug("storing server entry %s for user %s, key %s at timestamp %s" % (reading, user, key, ts))
+  currEntry = createEntry(user, key, ts, reading)
+  get_server_stats_db().insert(currEntry)
+
+def storeResultEntry(user, key, ts, reading):
+  logging.debug("storing result entry %s for user %s, key %s at timestamp %s" % (reading, user, key, ts))
+  currEntry = createEntry(user, key, ts, reading)
+  get_result_stats_db().insert(currEntry)
 
 def getClientMeasurementCount(readings):
   retSum = 0

CFC_WebApp/main/tripManager.py

@@ -123,8 +123,19 @@ def queryUnclassifiedSections(uuid):
                                                  {'user_id':user_uuid},
                                                  {'confirmed_mode': ''},
                                                  { 'type': 'move' }]})
-    logging.debug('Unsec.count = %s' % unclassifiedSections.count())
-    logging.debug('Total Unsec.count = %s' % totalUnclassifiedSections.count())
+
+    unclassifiedSectionCount = unclassifiedSections.count()
+    totalUnclassifiedSectionCount = totalUnclassifiedSections.count()
+
+    logging.debug('Unsec.count = %s' % unclassifiedSectionCount)
+    logging.debug('Total Unsec.count = %s' % totalUnclassifiedSectionCount)
+    # Keep track of what percent of sections are stripped out.
+    # Sections can be stripped out for various reasons:
+    # - they are too old
+    # - they have enough confidence that above the magic threshold (90%) AND
+    # the client has requested stripping out
+    stats.storeEntry(user_uuid, "stats.TRIP_MGR_PCT_SHOWN", time.time(),
+            float(unclassifiedSectionCount)/totalUnclassifiedSectionCount)
     return unclassifiedSections
 
 def getUnclassifiedSections(uuid):

CFC_WebApp/tests/client_tests/TestDefault.py

@@ -26,6 +26,10 @@ def testCarbonFootprintStore(self):
         default.setCarbonFootprint(user, dummyCarbonFootprint)
         # recall that pymongo converts tuples to lists somewhere down the line
         self.assertEquals(default.getCarbonFootprint(user), list(dummyCarbonFootprint))
+    
+    def testGetCategorySum(self):
+        calcSum = default.getCategorySum({'walking': 1, 'cycling': 2, 'bus': 3, 'train': 4, 'car': 5})
+        self.assertEqual(calcSum, 15)
 
 if __name__ == '__main__':
     unittest.main()