Merge pull request #887 from helgi/configurable_deploy_timeout

feat(app): make deploy timeout configurable globally/per-app via DEIS_DEPLOY_TIMEOUT, default is 2 minutes

Author: helgi

rootfs/api/models/app.py

@@ -404,6 +404,9 @@ def _scale_pods(self, scale_types):
         # see if the app config has deploy batch preference, otherwise use global
         batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES)
 
+        # see if the app config has deploy timeout preference, otherwise use global
+        deploy_timeout = release.config.values.get('DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT)  # noqa
+
         # see if there is a global or app specific setting to specify Deployments usage
         deployments = bool(envs.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
 
@@ -430,7 +433,7 @@ def _scale_pods(self, scale_types):
                 'routable': routable,
                 'deployments': deployments,
                 'deploy_batches': batches,
-                'deploy_timeout': 120,  # 2 minutes
+                'deploy_timeout': deploy_timeout,
             }
 
             command = self._get_command(scale_type)
@@ -469,6 +472,9 @@ def deploy(self, release, force_deploy=False):
         # see if there is a global or app specific setting to specify Deployments usage
         deployments = bool(release.config.values.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
 
+        # see if the app config has deploy timeout preference, otherwise use global
+        deploy_timeout = release.config.values.get('DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT)  # noqa
+
         deployment_history = release.config.values.get('KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)  # noqa
 
         # deploy application to k8s. Also handles initial scaling
@@ -497,7 +503,7 @@ def deploy(self, release, force_deploy=False):
                 'healthcheck': release.config.healthcheck,
                 'routable': routable,
                 'deploy_batches': batches,
-                'deploy_timeout': 120,  # 2 minutes
+                'deploy_timeout': deploy_timeout,
                 'deployment_history_limit': deployment_history,
                 'deployments': deployments,
                 'release_summary': release.summary
@@ -672,6 +678,9 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         if release.build is None:
             raise DeisException('No build associated with this release to run this command')
 
+        # see if the app config has deploy timeout preference, otherwise use global
+        deploy_timeout = release.config.values.get('DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT)  # noqa
+
         # TODO: add support for interactive shell
         entrypoint, command = self._get_command_run(command)
 
@@ -686,6 +695,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             'registry': release.config.registry,
             'version': "v{}".format(release.version),
             'build_type': release.build.type,
+            'deploy_timeout': deploy_timeout
         }
 
         try:

rootfs/api/models/release.py

@@ -359,9 +359,12 @@ def _delete_release_in_scheduler(self, namespace, version):
             'version': version
         }
 
+        # see if the app config has deploy timeout preference, otherwise use global
+        deploy_timeout = self.config.values.get('DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT)  # noqa
+
         controllers = self._scheduler.get_rcs(namespace, labels=labels).json()
         for controller in controllers['items']:
-            self._scheduler.cleanup_release(namespace, controller)
+            self._scheduler.cleanup_release(namespace, controller, deploy_timeout)
 
         # remove secret that contains env vars for the release
         try:

rootfs/api/settings/production.py

@@ -265,6 +265,12 @@
 # Can also be overwritten on per app basis if desired
 DEIS_DEPLOY_BATCHES = os.environ.get('DEIS_DEPLOY_BATCHES', None)
 
+# For old style deploys (RCs) defines how long each batch
+# (as defined by DEIS_DEPLOY_BATCHES) can take before giving up
+# For Kubernetes Deployments it is part of the global timeout
+# where it roughly goes BATCHES * TIMEOUT = global timeout
+DEIS_DEPLOY_TIMEOUT = os.environ.get('DEIS_DEPLOY_TIMEOUT', 120)
+
 # If the k8s Deployments object should be used instead of ReplicationController
 DEIS_KUBERNETES_DEPLOYMENTS = bool(os.environ.get('DEIS_KUBERNETES_DEPLOYMENTS', False))
 

rootfs/scheduler/__init__.py

@@ -109,7 +109,7 @@ def log(self, namespace, message, level=logging.INFO):
 
     def deploy(self, namespace, name, image, command, **kwargs):  # noqa
         """Scale RC or Deployment depending on what's requested"""
-        self.deploy_timeout = kwargs.get('deploy_timeout', 120)
+        self.deploy_timeout = kwargs.get('deploy_timeout')
         if kwargs.get('deployments', False):
             self.deploy_deployment(namespace, name, image, command, **kwargs)
         else:
@@ -163,6 +163,7 @@ def deploy_rc(self, namespace, name, image, command, **kwargs):  # noqa
         routable = kwargs.get('routable', False)
         envs = kwargs.get('envs', {})
         port = envs.get('PORT', None)
+        timeout = kwargs.get('deploy_timeout')
 
         # Fetch old RC and create the new one for a release
         old_rc = self.get_old_rc(namespace, app_type)
@@ -199,23 +200,23 @@ def deploy_rc(self, namespace, name, image, command, **kwargs):  # noqa
                 self.log(namespace, 'scaling release {} to {} out of final {}'.format(
                     new_name, count, desired
                 ))
-                self._scale_rc(namespace, new_name, count)
+                self._scale_rc(namespace, new_name, count, timeout)
 
                 if old_rc:
                     old_name = old_rc["metadata"]["name"]
                     self.log(namespace, 'scaling old release {} from original {} to {}'.format(
                         old_name, desired, (desired-count))
                     )
-                    self._scale_rc(namespace, old_name, (desired-count))
+                    self._scale_rc(namespace, old_name, (desired-count), timeout)
         except Exception as e:
             # New release is broken. Clean up
 
             # Remove new release of the RC
-            self.cleanup_release(namespace, new_rc)
+            self.cleanup_release(namespace, new_rc, timeout)
 
             # If there was a previous release then bring that back
             if old_rc:
-                self._scale_rc(namespace, old_rc["metadata"]["name"], desired)
+                self._scale_rc(namespace, old_rc["metadata"]["name"], desired, timeout)
 
             raise KubeException(
                 'Could not scale {} to {}. '
@@ -226,20 +227,20 @@ def deploy_rc(self, namespace, name, image, command, **kwargs):  # noqa
 
         # New release is live and kicking. Clean up old release
         if old_rc:
-            self.cleanup_release(namespace, old_rc)
+            self.cleanup_release(namespace, old_rc, timeout)
 
         # Make sure the application is routable and uses the correct port
         # Done after the fact to let initial deploy settle before routing
         # traffic to the application
         self._update_application_service(namespace, name, app_type, port, routable)
 
-    def cleanup_release(self, namespace, controller):
+    def cleanup_release(self, namespace, controller, timeout):
         """
         Cleans up resources related to an application deployment
         """
         # Deployment takes care of this in the API, RC does not
         # Have the RC scale down pods and delete itself
-        self._scale_rc(namespace, controller['metadata']['name'], 0)
+        self._scale_rc(namespace, controller['metadata']['name'], 0, timeout)
         self.delete_rc(namespace, controller['metadata']['name'])
 
         # Remove stray pods that the scale down will have missed (this can occassionally happen)
@@ -302,10 +303,10 @@ def _update_application_service(self, namespace, name, app_type, port, routable=
 
     def scale(self, namespace, name, image, command, **kwargs):
         """Scale RC or Deployment depending on what's requested"""
+        self.deploy_timeout = kwargs.get('deploy_timeout')
         if kwargs.get('deployments', False):
             self.scale_deployment(namespace, name, image, command, **kwargs)
         else:
-            self.deploy_timeout = kwargs.get('deploy_timeout', 120)
             self.scale_rc(namespace, name, image, command, **kwargs)
 
     def scale_deployment(self, namespace, name, image, command, **kwargs):
@@ -339,7 +340,7 @@ def scale_rc(self, namespace, name, image, command, **kwargs):
                     raise
 
         # let the scale failure bubble up
-        self._scale_rc(namespace, name, replicas)
+        self._scale_rc(namespace, name, replicas, kwargs.get('deploy_timeout'))
 
     def _build_pod_manifest(self, namespace, name, image, **kwargs):
         app_type = kwargs.get('app_type')
@@ -474,7 +475,13 @@ def run(self, namespace, name, image, entrypoint, command, **kwargs):
         # wait for run pod to start - use the same function as scale
         labels = manifest['metadata']['labels']
         containers = manifest['spec']['containers']
-        self._wait_until_pods_are_ready(namespace, containers, labels, desired=1)
+        self._wait_until_pods_are_ready(
+            namespace,
+            containers,
+            labels,
+            desired=1,
+            timeout=kwargs.get('deploy_timeout')
+        )
 
         try:
             # give pod 20 minutes to execute (after it got into ready state)
@@ -906,12 +913,11 @@ def _deploy_probe_timeout(self, timeout, namespace, labels, containers):
 
         return timeout
 
-    def _wait_until_pods_are_ready(self, namespace, containers, labels, desired):  # noqa
+    def _wait_until_pods_are_ready(self, namespace, containers, labels, desired, timeout):  # noqa
         # If desired is 0 then there is no ready state to check on
         if desired == 0:
             return
 
-        timeout = self.deploy_timeout
         timeout = self._deploy_probe_timeout(timeout, namespace, labels, containers)
         self.log(namespace, "waiting for {} pods in {} namespace to be in services ({}s timeout)".format(desired, namespace, timeout))  # noqa
 
@@ -957,7 +963,7 @@ def _wait_until_pods_are_ready(self, namespace, containers, labels, desired):  #
 
         self.log(namespace, "{} out of {} pods are in service".format(count, desired))  # noqa
 
-    def _scale_rc(self, namespace, name, desired):
+    def _scale_rc(self, namespace, name, desired, timeout):
         rc = self.get_rc(namespace, name).json()
 
         current = int(rc['spec']['replicas'])
@@ -976,7 +982,7 @@ def _scale_rc(self, namespace, name, desired):
         # Double check enough pods are in the required state to service the application
         labels = rc['metadata']['labels']
         containers = rc['spec']['template']['spec']['containers']
-        self._wait_until_pods_are_ready(namespace, containers, labels, desired)
+        self._wait_until_pods_are_ready(namespace, containers, labels, desired, timeout)
 
         # if it was a scale down operation, wait until terminating pods are done
         if int(desired) < int(current):