fix(scheduler): rollback scale / deploy when the desired number of pods can not be brought up in a timely manner

Prior to this if a deploy failed at bringing up pods then it would still scale down the old release instead of rolling back. This would cause users to basically have a broken app if the new pods for whatever reason stick around in Pending mode

Fixes #706

Author: helgi

rootfs/scheduler/__init__.py

@@ -993,6 +993,10 @@ def _wait_until_pods_are_ready(self, namespace, container, labels, desired):  #
             logger.info('timed out ({}s) waiting for pods to come up in namespace {}'.format(timeout, namespace))  # noqa
 
         logger.info("{} out of {} pods in namespace {} are in service".format(count, desired, namespace))  # noqa
+        if count != desired:
+            # raising to allow operations to rollback
+            raise KubeException('Not enough pods in namespace {} came into service. '
+                                '{} out of {}'.format(namespace, count, desired))
 
     def _scale_rc(self, namespace, name, desired):
         rc = self._get_rc(namespace, name).json()