1# (c) Stefan Countryman, 2019 

2 

3""" 

4Clean up the LLAMA output directory, rotating out-of-date auxilliary files out 

5of the way and archiving them on remote storage and setting event flag values 

6for matching events to the values specified in ``--flags``. You will need to 

7specify a pattern for matching events that need to be cleaned using the ``run`` 

8positional argument and the ``--downselect`` flag (see ``llama run -h`` help 

9documentation for details); for example, you can match events created more than 

1086400 seconds ago with ``--downselect sec_since_v0_gt=86400`` to 

11clean up test events older than a day. 

12""" 

13 

14import os 

15from datetime import datetime 

16from pathlib import Path 

17import logging 

18from llama.utils import OUTPUT_DIR, LOGDIR 

19from llama.run import Run 

20from llama.classes import ImmutableDict 

21 

22LOGGER = logging.getLogger(__name__) 

23TEST_ARCHIVE = os.path.join(OUTPUT_DIR, "test_event_archive") 

24LOGFILE = os.path.join(LOGDIR, "llama.dev.clean.log") 

25 

26 

27def set_flags_oldevents(run: Run, dryrun: bool = False, 

28 flags: dict = ImmutableDict({})): 

29 """ 

30 Set event flags for all events returned by ``run.events``. Apply any 

31 downselections you need to ``run`` before passing it. Use this to e.g. mark 

32 old events as no longer being able to upload files automatically. 

33 """ 

34 LOGGER.info("Setting flags to %s for events from %s", flags, run.rundir) 

35 for event in run.events: 

36 LOGGER.info("Setting flags for %s from %s to %s", event.eventdir, 

37 event.flags, flags) 

38 if not dryrun: 

39 event.flags.update(flags) 

40 

41 

42def cleanup_testevents(run: Run, archive_dir: str = None, 

43 dryrun: bool = False): 

44 """ 

45 Clean up all events returned by ``run.events`` and move them to a dated 

46 subdirectory of ``archive_dir`` (defaults to ``TEST_ARCHIVE``). You should 

47 apply any downselections to ``run`` so that ``run.events`` only returns 

48 test events older than the age you specify. If ``dryrun`` is ``True``, no 

49 action will be taken, but the intended actions will be logged. 

50 """ 

51 if archive_dir is None: 

52 archive_dir = TEST_ARCHIVE 

53 LOGGER.info("Archiving testevents from %s", run) 

54 for event in run.events: 

55 date = datetime.utcfromtimestamp(event.modification_time()) 

56 outdir = Path(archive_dir, f"{date.year}-{date.month:02d}", 

57 f"{date.day:02d}") 

58 LOGGER.info("Archiving %s -> %s", event.eventdir, outdir) 

59 if not dryrun: 

60 outdir.mkdir(parents=True, exist_ok=True) 

61 os.rename(event.eventdir, outdir/event.eventid)