diff --git a/Jenkinsfile b/Jenkinsfile index c63ad1d..4987c18 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,7 +30,7 @@ script { def skipFlag = params.SKIP_TESTS ? "--skip-tests" : "" // Call the python binary inside the venv directly - sh "./.venv/bin/python3 ./deployment --config /srv/jasonpoage.com/deployment.lua --branch ${env.TARGET_BRANCH} ${skipFlag}" + sh "./.venv/bin/python3 -u ./deployment --config /srv/jasonpoage.com/deployment.lua --branch ${env.TARGET_BRANCH} ${skipFlag}" } } } diff --git a/deployment/core/suite.py b/deployment/core/suite.py index 8c8b5eb..9a1f478 100644 --- a/deployment/core/suite.py +++ b/deployment/core/suite.py @@ -22,10 +22,10 @@ self.disable_dry_run() self.parser = None self._in_nix_shell = False - self.paths = None self.engine = None self.env = BuildEnv() self.args: dict = dict() + self.toml: dict = dict() self._owner = self self._parser() diff --git a/deployment/core/task_runner.py b/deployment/core/task_runner.py index 5ddf4ce..e7f2a6c 100755 --- a/deployment/core/task_runner.py +++ b/deployment/core/task_runner.py @@ -11,7 +11,7 @@ from core.tasks import ( GetDeploymentConfig, - VerifyConfigExists, + LoadServerConfig, YarnBuild, AtomicDeploy, HealthCheck, diff --git a/deployment/core/tasks.py b/deployment/core/tasks.py index ab83a00..a08aa10 100644 --- a/deployment/core/tasks.py +++ b/deployment/core/tasks.py @@ -24,32 +24,19 @@ config_path = self.get_arg("config") with open(config_path, "r") as f: - module = lua.execute(f.read()) - - # 2. Determine environment key from branch - # Mapping 'main' to 'release' as per lua schema - branch = self.get_arg("branch").split("/")[-1] - target_env = "release" if branch == "main" else branch - - if target_env not in module: - self.fail(f"Environment '{target_env}' not defined in {config_path}") - - # 3. Extract environment specific sub-table - cfg = module[target_env] + cfg = lua.execute(f.read()) # 4. Hydrate self.env self.env.lua_cfg = cfg # Store the lua object for functional calls later - self.env.app_name = module.app_name - self.env.repo = module.repo - self.env.timestamp_format = module.timestamp_format + self.env.app_name = cfg.app_name + self.env.repo = cfg.repo + self.env.timestamp_format = cfg.timestamp_format - self.env.deploy_path = Path(cfg.deploy_link) - self.env.service_name = cfg.service_name - self.env.config_file_source = Path(cfg.config_file) - self.env.retention_count = cfg.count - self.env.deploy_branch = branch + self.env.deploy_branch = self.get_arg("branch").split("/")[-1] + self.env.release = cfg.release + self.env.testing = cfg.testing - self.print(f"✅ Context hydrated for {self.env.app_name}:{target_env}") + self.print(f"✅ Context hydrated for {self.env.app_name}") # self.env.build_dir = Path(config.paths.build) return True @@ -65,50 +52,71 @@ def _run(self): # 1. Physical existence check - config_path = self.env.config_file_source - self.print(f" [CHECK] Verifying configuration: {config_path}") + self.env.toml["release"] = self.get_config("release") + self.env.toml["testing"] = self.get_config("testing") - if not os.path.exists(config_path): - self.fail( - f"CRITICAL: Configuration file not found at {config_path}. " - "Pipeline terminated to prevent application misbehavior." - ) + def get_config(self, env_type): + config_file = getattr(self.env, env_type).config_file + + self.print(f" [CHECK] Verifying {env_type} configuration: {config_file}") + + if not os.path.exists(config_file): + self.fail(f"CRITICAL: {env_type} config not found at {config_file}.") + # 1. Physical existence check # 2. Parse TOML for internal deployment metadata try: - with open(config_path, "rb") as f: + with open(config_file, "rb") as f: data = tomllib.load(f) - server = data.get("server", {}) + return { + "public": self.get_server_cfg(data, "public"), + "network": self.get_server_cfg(data, "network"), + } + + except Exception as e: + self.fail(f"FAILED to parse {env_type} TOML: ", e) + + def get_server_cfg(self, data, server_type): + try: + server = data.get(server_type) + print("Server", server) # 3. Hydrate self.env for HealthCheck and WaitForReadiness tasks - self.env.server_schema = server.get("schema") - self.env.server_address = server.get("address") - self.env.server_port = str(server.get("port")) - self.env.server_health_path = server.get("health_check") + config = { + "schema": server.get("schema"), + "domain": server.get("domain"), + "address": server.get("address"), + "port": str(server.get("port")), + } + print("config", config) + health_path = data.get("meta").get("health_check") + print("config", health_path) - # Construct the dynamic URI used by curl in later stages - self.env.test_endpoint_uri = ( - f"{self.env.server_schema}://{self.env.server_address}:" - f"{self.env.server_port}{self.env.server_health_path}" + if server_type == "network": + config["loc"] = config.get("address") + elif server_type == "public": + config["loc"] = config.get("domain") + + config["health_endpoint"] = ( + f"{config['schema']}://{config['loc']}:" + f"{config['port']}{health_path}" ) self.print( - f" [READY] Health check URI constructed: {self.env.test_endpoint_uri}" + f" [READY] {server_type} Health URI: {config['health_endpoint']}" ) + return config except Exception as e: - self.fail(f"FAILED to parse TOML at {config_path}: {e}") - - self.print(" [OK] Configuration verified and environment hydrated.") - return True + self.fail(f"FAILED to parse {server_type} TOML: ", e) class YarnBuild(SuiteTask): """Executes dependency installation and asset compilation""" _stage = Stage.BUILD - _deps = [GetDeploymentConfig, VerifyConfigExists] + _deps = [GetDeploymentConfig, LoadServerConfig] skip: bool = False def __init__(self, *args, **kwargs): @@ -116,20 +124,15 @@ self.name = "Running Yarn build process" def _run(self): - # Use a temporary build directory with -BUILDING suffix - # This is finalized in AtomicDeploy timestamp = time.strftime(self.env.timestamp_format) - self.env.release_dir = Path(self.env.lua_cfg.get_release_dir(timestamp)) - self.env.build_dir = self.env.release_dir.with_name( - self.env.release_dir.name + "-BUILDING" - ) - + self.env.release_dir = f"{Path(self.env.testing.deploy_link)}-{timestamp}" self.print(f" [BUILD] Target: {self.env.build_dir}") self.sh( f"git clone --branch {self.env.deploy_branch} {self.env.repo} {self.env.build_dir}" ) self.sh("git submodule update --init --recursive", cwd=self.env.build_dir) + self.sh("yarn config set enableGlobalCache false", cwd=self.env.build_dir) self.sh("yarn install", cwd=self.env.build_dir) self.sh("yarn combine:css", cwd=self.env.build_dir) return True @@ -147,43 +150,65 @@ self.name = "Executing atomic symlink swap" def _run(self): - # 1. Finalize the directory name (remove -BUILDING) - self.sh(f"mv {self.env.build_dir} {self.env.release_dir}") + # Determine success from the TestRunner flag + test_success = getattr(self.env, "test_success", False) - # 2. Atomic Symlink Swap - temp_link = self.env.deploy_path.with_name(self.env.deploy_path.name + "_tmp") - self.sh(f"ln -sfn {self.env.release_dir} {temp_link}") - self.sh(f"mv -Tf {temp_link} {self.env.deploy_path}") + # Select appropriate Lua config table + cfg = self.env.release if test_success else self.env.testing - # 3. Restart Service - self.sh(f"sudo systemctl restart {self.env.service_name}") + # Generate the versioned directory path using Lua function + # Note: Use the actual formatted timestamp, not the format string + timestamp = time.strftime(self.env.timestamp_format) + final_release_dir = Path(cfg.get_release_dir(timestamp)) - self.print(f"🚀 Deployed to {self.env.deploy_path} -> {self.env.release_dir}") + self.print(f" [MOVE] Finalizing build to: {final_release_dir}") + + # 1. Finalize the directory (Rename from -BUILDING to versioned path) + self.sh(f"mv {self.env.build_dir} {final_release_dir}") + + # 2. Atomic Symlink Swap - ONLY if tests passed + if test_success: + deploy_link = Path(cfg.deploy_link) + # Create a temporary symlink name in the same parent directory + temp_link = deploy_link.with_name(deploy_link.name + "_tmp") + + self.print( + f" [LINK] Swapping symlink: {deploy_link} -> {final_release_dir}" + ) + + # Create temporary symlink pointing to the new version + self.sh(f"ln -sfn {final_release_dir} {temp_link}") + + # Atomic rename of the symlink itself (overwrites the old link) + self.sh(f"mv -Tf {temp_link} {deploy_link}") + + # Restart service + self.sh(f"sudo systemctl restart {cfg.service_name}") + else: + self.print(" [SKIP] Test failure detected. Symlink swap bypassed.") + self.print(f" [INFO] Artifact preserved at: {final_release_dir}") + return True class HealthCheck(SuiteTask): - """Polls the local service endpoint to verify readiness""" + """Polls the local production service endpoint""" _stage = Stage.DEPLOY _deps = [AtomicDeploy] - skip: bool = False def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.name = "Verifying service health" def _run(self): - self.msg(self.name) - if self.do_dry_run: + # Base run handles dry_run check already + uri = self.env.toml["release"]["network"]["health_endpoint"] + + status = self.poll_health_endpoint(uri, label="Production Service") + if self.do_dry_run(): return - for i in range(15): - res = subprocess.run( - ["curl", "-s", "-I", self.env.test_endpoint_uri], - capture_output=True, - ) - if res.returncode == 0: - self.print("✅ Service is healthy") - return True - time.sleep(2) - self.fail("Service failed health check after 30 seconds") + if not status: + self.fail(f"Production service failed health check at {uri}") + + return True diff --git a/deployment/core/tests.py b/deployment/core/tests.py index 547b855..40fc505 100644 --- a/deployment/core/tests.py +++ b/deployment/core/tests.py @@ -16,24 +16,22 @@ super().__init__(*args, **kwargs) def _run(self): - if self._owner.args.get("skip_tests") and not self.env.meta.get( - "enforce_testing" - ): + if self._owner.args.get("skip_tests") and not self.get_arg("enforce_testing"): self.print(" [SKIP] Skipping per user request.") return True self.print(f" [EXEC] Starting app in {self.env.build_dir}") # Stop existing service if it's hogging the port - self.sh(f"sudo systemctl stop {self.env.service_name} || true") + self.sh(f"sudo systemctl stop {self.env.testing.service_name} || true") # Start background process and record PID - cmd = f"nohup yarn run prod >> '{self.env.meta.server_log_file}' 2>&1 & echo $! > '{self.env.pidfile}'" + cmd = f"nohup yarn run prod >> '{self.env.test_log}' 2>&1 & echo $! > '{self.env.pidfile}'" self.sh(cmd, cwd=self.env.build_dir) return True class WaitForReadiness(SuiteSubTask): - """Polls the health endpoint until the test server is responsive""" + """Polls the health endpoint of the TEST instance""" _stage = Stage.TEST _deps = [StartTestApp] @@ -43,28 +41,22 @@ self.name = "Wait for Service Readiness" def _run(self): - if self._owner.args.get("skip_tests") and not self.env.meta.get( - "enforce_testing" + if self._owner.args.get("skip_tests") and not self.get_arg( + "enforce_testing", True ): return True - uri = self.env.test_endpoint_uri - self.print(f" [POLL] Waiting for {uri}...") - # if self.do_dry_run(): - # return + uri = self.env.toml["testing"]["network"]["health_endpoint"] - for _ in range(15): - # Check for 200 OK - try: - res = self.sh(f"curl -s -I {uri} | grep '200 OK'") - if res: - self.print(" [OK] Service is UP.") - return True - except Exception: - time.sleep(2) + status = self.poll_health_endpoint(uri, label="Test Service") + if self.do_dry_run(): + return + if not status: + # If the poll fails, we cat the log as requested before failing + self.sh(f"cat '{self.env.test_log}'") + self.fail(f"Test service at {uri} failed to start.") - self.sh(f"cat '{self.env.meta.server_log_file}'") - self.fail(f"Service at {uri} failed to start within 30s.") + return True class RunMochaTests(SuiteSubTask): @@ -78,9 +70,7 @@ super().__init__(*args, **kwargs) def _run(self): - if self._owner.args.get("skip_tests") and not self.env.meta.get( - "enforce_testing" - ): + if self._owner.args.get("skip_tests") and not self.get_arg("enforce_testing"): return True self.print(" [RUN] npm run test:postreceive") @@ -124,7 +114,7 @@ def _run(self): # 1. Check if we should even be here skip_param = self.args.get("skip_tests", False) - enforced = (self.env.meta.enforce_testing,) + enforced = self.get_arg("enforce_testing") if skip_param and not enforced: self.print(" [SKIP] Integration tests bypassed by user flag.") @@ -157,5 +147,4 @@ self.print(" [CLEAN] Ensuring test environment teardown...") cleanup = StopTestApp(parent=self, owner=self._owner) cleanup.run() - - return success + self.env.test_success = success diff --git a/deployment/lib/errors.py b/deployment/lib/errors.py index bbe22c1..e4e64e1 100644 --- a/deployment/lib/errors.py +++ b/deployment/lib/errors.py @@ -1,4 +1,5 @@ import sys +import traceback class SuiteError(Exception): @@ -7,6 +8,7 @@ ): super().__init__(*args, **kwargs) parent.dump_print_queue() + traceback.print_stack() print(*args, **kwargs) if code is not None: @@ -26,3 +28,4 @@ raise RuntimeError(*args, **kwargs) else: print(*args, **kwargs) + traceback.print_stack() diff --git a/deployment/lib/printer.py b/deployment/lib/printer.py index 1ca254e..b5df1b8 100755 --- a/deployment/lib/printer.py +++ b/deployment/lib/printer.py @@ -39,7 +39,7 @@ Printer._queue.append(payload) else: - print(*args, **kwargs) + print(*args, flush=True, **kwargs) def flush(self): for args, kwargs in Printer._queue: @@ -57,6 +57,7 @@ f.write(json.dumps(line)) except Exception as e: print(e.with_traceback) + raise e def _msg_prefix(self): # Format: [ID] for main tasks, [ID.Sub] for subtasks diff --git a/deployment/lib/task_types.py b/deployment/lib/task_types.py index ba8128e..ade65d9 100755 --- a/deployment/lib/task_types.py +++ b/deployment/lib/task_types.py @@ -1,3 +1,4 @@ +import time import threading import os import sys @@ -59,10 +60,7 @@ except: pass if cwd is None: - try: - cwd = os.getcwd() - except: - pass + cwd = os.getcwd() self._cwd = cwd self._owner = owner @@ -149,19 +147,20 @@ raise TaskError(self, critical=critical, *args, **kwargs) - def sh(self, cmd: str, cwd: Path | None = None, graceful=False, dry_run=None): + def sh( + self, cmd: str, cwd: Path | None = None, handle_exception=True, dry_run=None + ): """Helper to run shell commands within the project context.""" + cwd = str(cwd or os.getcwd()) + self.print(f" [CWD] {cwd}") self.msg(f" [EXEC] {cmd}") - if self.do_dry_run and dry_run is not False: + if self.do_dry_run() and dry_run is not False: return try: - # E: Instance of 'SuiteTask' has no 'paths' member - return subprocess.run( - cmd, shell=True, check=True, cwd=str(cwd or os.getcwd()) - ) - except Exception as e: - if graceful: + return subprocess.run(cmd, shell=True, check=True, cwd=cwd) + except subprocess.CalledProcessError as e: + if handle_exception: self.fail(e) raise Exception(e) @@ -228,6 +227,41 @@ def get_stage(self): return self._stage + def deps_loaded(self): + if isinstance(self, SuiteSubTask): + return True + from core.task_runner import TaskRunner + + return TaskRunner.is_loaded(self._deps) + + def poll_health_endpoint(self, uri, retries=15, delay=2, label="Service"): + """Shared polling logic for verifying service availability""" + self.print(f" [POLL] Verifying {label} Health: {uri}") + + if self.do_dry_run(): + retries = 0 + + for _ in range(retries): + try: + # Use sh to maintain consistency in logs/dry-runs + # We use graceful=False but handle the boolean return in the loop + res = self.sh( + f"curl -s -I {uri} | grep '200 OK'", handle_exception=False + ) + + if res and res.returncode == 0: + self.print(f" [OK] {label} is healthy.") + return True + else: + self.print("Got result :", res) + except Exception as e: + + self.print(f" [WAIT] {label} not ready... {e}") + + time.sleep(delay) + + return False + class SuiteSubTask(SuiteTask): _owner: "TDDSuite" @@ -245,8 +279,6 @@ self.attach_printer(self._owner) - self.paths = self._owner.paths - def msg(self, *args, **kwargs): """Standardized message logger.""" SuiteSubTask.inc_count() @@ -255,6 +287,8 @@ @staticmethod def inc_count(): + + print(SuiteSubTask._sub_counter) SuiteSubTask._sub_counter[SuiteTask._global_counter] += 1 @staticmethod diff --git a/deployment/lib/types.py b/deployment/lib/types.py index ddc4c89..e2ea767 100644 --- a/deployment/lib/types.py +++ b/deployment/lib/types.py @@ -22,13 +22,14 @@ release_dir: Path test_endpoint_uri: str pidfile: Path = Path() + test_log: Path = Path() + toml: dict = {} def __init__(self, timestamp_format: str | None = None): self.workspace: Path = Path() self.timestamp: str = "" self.deploy_branch: str = "" self.deploy_path: Path = Path() - self.build_dir: Path = Path() self.service_name: str = "" self.release_dir: Path = Path() self.server_schema = "http" @@ -40,3 +41,5 @@ self.timestamp_format = timestamp_format self.workspace = Path(os.getenv("WORKSPACE", self.root_dir)) self.build_dir = self.workspace / "build" + + self.test_log: Path = self.build_dir / "test_log.log" diff --git a/deployment/main.py b/deployment/main.py index cf9fe99..28858d4 100644 --- a/deployment/main.py +++ b/deployment/main.py @@ -7,7 +7,7 @@ def main(): runner = DeploymentSuite() - exit_code = None + exit_code = 0 try: runner.run() @@ -19,7 +19,8 @@ exit_code = 0 except Exception as e: print(f"❌ Deployment Failed at: {e.with_traceback(e.__traceback__)}") - raise exit_code = 1 runner.dump_print_queue() - sys.exit(exit_code or 1) + if exit_code != 0: + print(exit_code) + sys.exit(exit_code or 1) diff --git a/deployment/prototype.py b/deployment/prototype.py deleted file mode 100755 index b591d4a..0000000 --- a/deployment/prototype.py +++ /dev/null @@ -1,177 +0,0 @@ -GIT_REPO = 'ssh://git@git.jasonpoage.vpn:29418/jason/express-blog.git' -DEPLOY_BASE = '/srv/jasonpoage.com' -YARN_ENABLE_GLOBAL_CACHE = 'false' -YARN_CACHE_FOLDER = '/var/cache/jenkins/yarn' -CREDENTIALS_ID = '08a57452-477d-4aa6-86c6-242553660b3f' - - - -options { - timestamps() -} - - -parameters { - string(name: 'branch', defaultValue: 'refs/heads/main', description: 'Branch ref from webhook') - string(name: 'oldrev', defaultValue: '', description: 'old rev') - string(name: 'newrev', defaultValue: '', description: 'new rev') - - booleanParam(name: 'SKIP_TESTS', defaultValue: true, description: 'Skip all testing') -} -class test_runner: - def init(): - print('Init') - if params.branch?.startsWith("refs/heads/"): - DEPLOY_BRANCH = params.branch.replaceFirst(/^refs\/heads\//, '') - else: - print(f "Invalid branch ref: '{params.branch}'") - - - print( "==== DEBUG: Branch Param ====") - print ("params.branch: '{params.branch}'") - print ("DEPLOY_BRANCH: '{DEPLOY_BRANCH}'") - - TIMESTAMP = sh(script: "date +%Y%m%d-%H%M%S", returnStdout: true).trim() - LOG_DIR = f"{DEPLOY_BASE}/deployments/logs" - SERVER_LOG_FILE = f"{LOG_DIR}/server/server-{TIMESTAMP}.log" - TEST_LOGS_FILE = f"{LOG_DIR}/test-results/test-" - BUILD_DIR = f"{WORKSPACE}/build" - PIDFILE = f"{BUILD_DIR}/test.pid" - ENV_FILE = f"{DEPLOY_BASE}/env/{DEPLOY_BRANCH}.env" - SERVICE_NAME = f"express-blog@{DEPLOY_BRANCH}.service" - DEPLOY_PATH = f"{DEPLOY_BASE}/deployments/blog-{DEPLOY_BRANCH}" - - - if params.oldrev?.trim() && params.newrev?.trim(): - OLD_REV = params.oldrev - NEW_REV = params.newrev - else : - OLD_REV = sh(script: 'git rev-parse HEAD~1', returnStdout: true).trim() - NEW_REV = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() - - print (f"==== DEBUG: Revisions ====") - print (f"params.oldrev: '{params.oldrev}'") - print (f"params.newrev: '{params.newrev}'") - print (f"Old revision: {OLD_REV}") - print (f"New revision: {NEW_REV}") - sh (f"mkdir -p '{LOG_DIR}/server' '{LOG_DIR}/test-results'") - def checkout () : - print('Checkout') - checkout([$class: 'GitSCM', - branches: [[name: "*/{DEPLOY_BRANCH}"]], - userRemoteConfigs: [[ - url: GIT_REPO, - credentialsId: CREDENTIALS_ID - ]] - ]) - - def validate_branch(): - print('Validate Branch') - steps { - script { - def allowed = ['testing', 'staging', 'main', 'production'] - if (!allowed.contains(DEPLOY_BRANCH)) { - fail "Branch '{DEPLOY_BRANCH}' is not allowed for deployment." - } - } - } - - def clone_build_dir(): - print('Clone to Build Dir') - sh (f"git clone --branch '{DEPLOY_BRANCH}' '{GIT_REPO}' '{BUILD_DIR}'") - - def build(): - print('Build') - dir(f"{BUILD_DIR}") { - sh """ - git submodule update --init --recursive - yarn - yarn combine:css - """ - - def start_app(): - print('Start Application for Test') { - if not params.SKIP_TESTS : - dir(BUILD_DIR) { - sh """ - sudo systemctl stop {SERVICE_NAME} || true - corepack enable - nohup yarn run prod >> '{SERVER_LOG_FILE}' 2>&1 & - echo \$! > '{PIDFILE}' - """ - - def wait_for_service(): - print('Wait for Service Readiness') - if not params.SKIP_TESTS : - def timeout = 30 - def elapsed = 0 - def success = false - while elapsed < timeout : - def result = sh(script: f"curl --max-time 2 --silent --fail '\{SERVER_SCHEMA}://\{SERVER_DOMAIN}/health -I' > /dev/null || true", returnStatus: true) - if result == 0: - success = true - break - sleep 1 - elapsed += 1 - if not success : - sh f"cat '{SERVER_LOG_FILE}'" - fail f"Service did not become available within {timeout}s." - } - } - } - - def run_tests(): - print('Run Tests') - if not params.SKIP_TESTS : - def testStatus = sh(script: f"cd '{BUILD_DIR}' && npm run test:postreceive", returnStatus: true) - archiveArtifacts artifacts: f"{TEST_LOGS_FILE}*", onlyIfSuccessful: false - if testStatus != 0: - sh f""" - kill \$(cat '{PIDFILE}') || true - cat '{SERVER_LOG_FILE}' - """ - fail( "Tests failed for branch {DEPLOY_BRANCH}") - } - } - - def kill_test_server(): - """systemctl stop test server""" - print('Stop Test App') - if not params.SKIP_TESTS : - sh ("kill \$(cat '{PIDFILE}') || true") - def deploy(): - print('Deploy') - # 1. Create the new release directory - releaseDir = f"{DEPLOY_BASE}/releases/blog-{DEPLOY_BRANCH}-{TIMESTAMP}" - sh( f"mkdir -p {releaseDir}") - - # 2. Sync the finished build to the release directory - print(f"Deploying build to {releaseDir}") - sh( f"rsync -a --delete '{BUILD_DIR}/' '{releaseDir}/'") - - # 3. Atomically flip the symlink - # We use 'ln -sfn' to overwrite the existing link to the new path - sh (f"ln -sfn '{releaseDir}' '{DEPLOY_PATH}'") - - # 4. Cleanup old releases (Keep only last 5) - dir(f"{DEPLOY_BASE}/releases") { - sh f"ls -1t | grep 'blog-{DEPLOY_BRANCH}' | tail -n +6 | xargs rm -rf || true" - } - def restart_production_server(): - print('Restart Service') - sh f"sudo systemctl restart {SERVICE_NAME}" - def verify_service(): - print("VerifyService") - timeout = 30 - elapsed = 0 - success = false - while (elapsed < timeout): - result = sh(script: f"curl --max-time 2 --silent --fail '{SERVER_SCHEMA}://{SERVER_DOMAIN}/health -I' > /dev/null || true", returnStatus: true) - if result == 0: - success = true - break - sleep 1 - elapsed += 1 - if not success: - print(SERVER_LOG_FILE) - self.fail( f"Service did not become available within {timeout}s.")