diff options
Diffstat (limited to 'auto_delete_nightly_test_data.py')
| -rwxr-xr-x | auto_delete_nightly_test_data.py | 173 |
1 files changed, 35 insertions, 138 deletions
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index 0dd2dba8..06d02067 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be @@ -10,88 +9,17 @@ __author__ = "shenhan@google.com (Han Shen)" import argparse -import datetime import os from pathlib import Path -import re import shutil import stat import sys import time import traceback -from typing import Callable +from typing import Callable, List from cros_utils import command_executer from cros_utils import constants -from cros_utils import misc - - -DIR_BY_WEEKDAY = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") -NIGHTLY_TESTS_WORKSPACE = os.path.join( - constants.CROSTC_WORKSPACE, "nightly-tests" -) - - -def CleanNumberedDir(s, dry_run=False): - """Deleted directories under each dated_dir.""" - chromeos_dirs = [ - os.path.join(s, x) - for x in os.listdir(s) - if misc.IsChromeOsTree(os.path.join(s, x)) - ] - ce = command_executer.GetCommandExecuter(log_level="none") - all_succeeded = True - for cd in chromeos_dirs: - if misc.DeleteChromeOsTree(cd, dry_run=dry_run): - print(f"Successfully removed chromeos tree {cd!r}.") - else: - all_succeeded = False - print(f"Failed to remove chromeos tree {cd!r}, please check.") - - if not all_succeeded: - print("Failed to delete at least one chromeos tree, please check.") - return False - - ## Now delete the numbered dir Before forcibly removing the directory, just - ## check 's' to make sure it matches the expected pattern. A valid dir to be - ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. - valid_dir_pattern = ( - "^" + NIGHTLY_TESTS_WORKSPACE + "/(" + "|".join(DIR_BY_WEEKDAY) + ")" - ) - if not re.search(valid_dir_pattern, s): - print( - f"Trying to delete an invalid dir {s!r} (must match " - f"{valid_dir_pattern!r}), please check." - ) - return False - - cmd = f"rm -fr {s}" - if dry_run: - print(cmd) - else: - if ( - ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) - == 0 - ): - print(f"Successfully removed {s!r}.") - else: - all_succeeded = False - print(f"Failed to remove {s!r}, please check.") - return all_succeeded - - -def CleanDatedDir(dated_dir, dry_run=False): - # List subdirs under dir - subdirs = [ - os.path.join(dated_dir, x) - for x in os.listdir(dated_dir) - if os.path.isdir(os.path.join(dated_dir, x)) - ] - all_succeeded = True - for s in subdirs: - if not CleanNumberedDir(s, dry_run): - all_succeeded = False - return all_succeeded def ProcessArguments(argv): @@ -136,10 +64,10 @@ def RemoveAllSubdirsMatchingPredicate( try: dir_entries = list(base_dir.iterdir()) except FileNotFoundError as e: - # We get this if the directory itself doesn't exist. Since we're cleaning - # tempdirs, that's as good as a success. Further, the prior approach here - # was using the `find` binary, which exits successfully if nothing is - # found. + # We get this if the directory itself doesn't exist. Since we're + # cleaning tempdirs, that's as good as a success. Further, the prior + # approach here was using the `find` binary, which exits successfully + # if nothing is found. print(f"Error enumerating {base_dir}'s contents; skipping removal: {e}") return 0 @@ -149,8 +77,8 @@ def RemoveAllSubdirsMatchingPredicate( continue try: - # Take the stat here and use that later, so we only need to check for a - # nonexistent file once. + # Take the stat here and use that later, so we only need to check + # for a nonexistent file once. st = file.stat() except FileNotFoundError: # This was deleted while were checking; ignore it. @@ -176,30 +104,32 @@ def RemoveAllSubdirsMatchingPredicate( shutil.rmtree(file, onerror=OnError) - # Some errors can be other processes racing with us to delete things. Don't - # count those as an error which we complain loudly about. + # Some errors can be other processes racing with us to delete things. + # Don't count those as an error which we complain loudly about. if this_iteration_had_errors: if file.exists(): had_errors = True else: print( - f"Discarding removal errors for {file}; dir was still removed." + f"Discarding removal errors for {file}; dir was still " + "removed." ) return 1 if had_errors else 0 def IsChromeOsTmpDeletionCandidate(file_name: str): - """Returns whether the given basename can be deleted from a chroot's /tmp.""" + """Returns whether the given basename can be deleted from chroot's /tmp.""" name_prefixes = ( "test_that_", "cros-update", "CrAU_temp_data", + # This might seem a bit broad, but after using custom heuristics for a + # while, `/tmp` was observed to have >75K files that matched all sorts + # of different `tmp.*` name patterns. Just remove them all. + "tmp", ) - if any(file_name.startswith(x) for x in name_prefixes): - return True - # Remove files that look like `tmpABCDEFGHI`. - return len(file_name) == 9 and file_name.startswith("tmp") + return any(file_name.startswith(x) for x in name_prefixes) def CleanChromeOsTmpFiles( @@ -238,47 +168,47 @@ def CleanChromeOsImageFiles( try: shutil.rmtree(subdir_path) print( - "Successfully cleaned chromeos image autotest directories " - f"from {subdir_path!r}." + "Successfully cleaned chromeos image autotest " + f"directories from {subdir_path!r}." ) except OSError: print( - "Some image autotest directories were not removed from " - f'"{subdir_path}".' + "Some image autotest directories were not " + f'"removed from {subdir_path}".' ) errors += 1 return errors -def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): +def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False) -> int: """Delete temporaries, images under crostc/chromeos.""" chromeos_chroot_tmp = os.path.join( - constants.CROSTC_WORKSPACE, "chromeos", "chroot", "tmp" + constants.CROSTC_WORKSPACE, "chromeos", "out", "tmp" ) # Clean files in tmp directory rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run) # Clean image files in *-tryjob directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-tryjob", days_to_preserve, dry_run ) # Clean image files in *-release directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-release", days_to_preserve, dry_run ) # Clean image files in *-pfq directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-pfq", days_to_preserve, dry_run ) # Clean image files in *-llvm-next-nightly directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-llvm-next-nightly", days_to_preserve, dry_run ) return rv -def CleanOldCLs(days_to_preserve="1", dry_run=False): +def CleanOldCLs(days_to_preserve: str = "1", dry_run: bool = False) -> int: """Abandon old CLs created by automation tooling.""" ce = command_executer.GetCommandExecuter() chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, "chromeos") @@ -305,15 +235,7 @@ def CleanOldCLs(days_to_preserve="1", dry_run=False): def CleanChromeTelemetryTmpFiles(dry_run: bool) -> int: - tmp_dir = ( - Path(constants.CROSTC_WORKSPACE) - / "chromeos" - / ".cache" - / "distfiles" - / "chrome-src-internal" - / "src" - / "tmp" - ) + tmp_dir = Path(constants.CROSTC_WORKSPACE) / "chrome" / "src" / "tmp" return RemoveAllSubdirsMatchingPredicate( tmp_dir, days_to_preserve=0, @@ -323,47 +245,22 @@ def CleanChromeTelemetryTmpFiles(dry_run: bool) -> int: ) -def Main(argv): +def Main(argv: List[str]) -> int: """Delete nightly test data directories, tmps and test images.""" options = ProcessArguments(argv) - # Function 'isoweekday' returns 1(Monday) - 7 (Sunday). - d = datetime.datetime.today().isoweekday() - # We go back 1 week, delete from that day till we are - # options.days_to_preserve away from today. - s = d - 7 - e = d - int(options.days_to_preserve) - rv = 0 - for i in range(s + 1, e): - if i <= 0: - ## Wrap around if index is negative. 6 is from i + 7 - 1, because - ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7. - dated_dir = DIR_BY_WEEKDAY[i + 6] - else: - dated_dir = DIR_BY_WEEKDAY[i - 1] - - rv += ( - 0 - if CleanDatedDir( - os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir), - options.dry_run, - ) - else 1 - ) - ## Clean temporaries, images under crostc/chromeos - rv2 = CleanChromeOsTmpAndImages( + rv = CleanChromeOsTmpAndImages( int(options.days_to_preserve), options.dry_run ) # Clean CLs that are not updated in last 2 weeks. - rv3 = CleanOldCLs("14", options.dry_run) + rv |= CleanOldCLs("14", options.dry_run) # Clean telemetry temporaries from chrome source tree inside chroot. - rv4 = CleanChromeTelemetryTmpFiles(options.dry_run) + rv |= CleanChromeTelemetryTmpFiles(options.dry_run) - return rv + rv2 + rv3 + rv4 + return 1 if rv else 0 if __name__ == "__main__": - retval = Main(sys.argv[1:]) - sys.exit(retval) + sys.exit(Main(sys.argv[1:])) |