Coverage for /opt/anaconda/lib/python3.7/site-packages/llama/vetoes.py : 57%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# (c) Stefan Countryman 2019
3"""
4A mechanism for vetoing steps in an analysis.
5"""
7import os
8import datetime
9import uuid
10import json
11import logging
12import functools
13import shutil
14from collections import namedtuple
15from llama.flags import FlagsMixin
16from llama.filehandler.classes import GenerationResult
18LOGGER = logging.getLogger(__name__)
20# this is just a function, so don't give it an uppercase constant var name
21# pylint: disable=invalid-name
22utcnow = datetime.datetime.utcnow
25# define some common veto use cases
26def eventdir_path_contains_string_scratch(eventdir):
27 """If the path to the trigger directory contains the substring 'SCRATCH'
28 (case-insensitive), this veto is triggered (returns True)."""
29 return eventdir.upper().find('SCRATCH') != -1
32def eventdir_path_contains_string_manual(eventdir):
33 """If the path to the trigger directory contains the substring 'MANUAL'
34 (case-insensitive), this veto is triggered (returns True)."""
35 return eventdir.upper().find('MANUAL') != -1
38def eventdir_path_contains_string_test(eventdir):
39 """If the path to the trigger directory contains the substring 'TEST'
40 (case-insensitive), this veto is triggered (returns True)."""
41 return eventdir.upper().find('TEST') != -1
44def eventdir_path_contains_string_injection(eventdir):
45 """If the path to the trigger directory contains the substring 'INJECTION'
46 (case-insensitive), this veto is triggered (returns True)."""
47 return eventdir.upper().find('INJECTION') != -1
50# this docstring component is also used in FileHandler, so explicitly make sure
51# that it's the same string in both.
52VETO_CHECKS_DOCSTRING="""
53An iterable of N length-2 iterables of the form:
55 ((veto1, action1), ..., (vetoN, actionN))
57matching ``veto`` functions to ``action`` functions.
59``veto`` functions are functions that take ``eventdir`` as
60arguments and return ``True`` if that particular veto criterion has
61been met (for example, a ``veto`` could return ``True`` if a trigger
62has somehow been marked as a test trigger).
64``action`` functions are callables that indicate some action that
65should be taken to handle the ``veto``. ``action`` can also be ``None``
66for the default behavior: to mark the set of filenames (specified
67by ``manifest``) for this trigger as vetoed, which would prevent
68those files from being generated.
70Alternatively, you can also use an actual function for ``action``
71to do something more subtle than just vetoing the file (e.g.
72delaying file generation, advanced error logging, dummy testing,
73etc; this could even involve an alternative file generation method
74for test cases or edge cases). The ``veto`` functions will be run in
75order and the first to return ``True`` will have its ``action``
76executed; the rest will be ignored.
78*NOTE* that if you specify an action, the file will NOT be
79vetoed by default (since your veto-handling might involve something
80like delaying file generation or generating it through some other
81method, both of which are cases where you are not actually trying
82to prevent the file from being generated). If you want to fully
83veto file generation, you will have to manually call this
84``VetoHandler`` instance at the end of your ``action`` to mark these
85files as PERMANENTLY vetoed and prevent file generation.
87A ``VetoException`` WILL always be raised, however, since the
88called ``action`` is supposed to cancel any *IMMEDIATE* file
89generation by the default method (even if later attempts are still
90allowed).
91"""
94class VetoException(IOError):
95 """
96 Indicates that a FileHandler's generation was vetoed and should proceed
97 no futher at this time. This is a benign exception that should be caught at
98 file generation time.
99 """
102VetoHandlerTuple = namedtuple('VetoHandlerTuple', ('eventdir', 'manifest',
103 'vetoes'))
106def generation_dir_copy_vetoes(vetoed, tmpdir):
107 """
108 Copy any vetofiles for the ``vetoed`` filehandler from the temporary
109 directory ``tmpdir`` back to the eventdir. Vital for cleaning up after a
110 failed generation attempt (in which the filehandler manifest would NOT be
111 copied because the files should never have been generated; see
112 _generation_dir_copy_manifest).
113 """
114 veto_src_dest_paths = [
115 (os.path.join(tmpdir, f), os.path.join(vetoed.eventdir, f))
116 for f in vetoed.veto.vetofilenames
117 ]
118 for src, dest in veto_src_dest_paths:
119 if os.path.isfile(src):
120 LOGGER.info("Found vetofile %s in the temp directory; copying"
121 "back to the real eventdir.", src)
122 try:
123 os.link(src, dest)
124 except OSError:
125 try:
126 shutil.copy(src, dest)
127 except shutil.SameFileError:
128 pass
131class VetoHandler(VetoHandlerTuple, FlagsMixin):
132 """
133 A class that performs veto checks to see whether a FileHandler for a
134 *specific* trigger should be vetoed. Instances can be called like functions
135 to mark a FileHandler as vetoed.
136 """
138 def __new__(cls, eventdir, manifest, vetoes):
139 """See ``__init__`` for signature. Coerces arguments into the proper
140 (mutable) classes."""
141 return VetoHandlerTuple.__new__(cls, eventdir, tuple(manifest),
142 tuple(vetoes))
144 def __init__(self, eventdir, manifest, vetoes):
145 """Specify which eventdir and filenames need to be checked for
146 vetoes.
148 Parameters:
149 -----------
150 eventdir : str
151 The path to the directory in which the possibly-vetoed files would
152 be stored if generated.
153 manifest : array-like
154 An iterable of file names for the files which might need to be
155 vetoed. Should just be the same as the manifest from the
156 FileHandler instance using this VetoHandler.
157 vetoes : array-like
158 """
159 # NOTE: This is a no-op since tuples don't use __init__. I'm putting
160 # __init__ here just to store the docstring and __init__ signature in a
161 # way accessible to ipython.
163 # pylint: disable=no-member
164 __init__.__doc__ += 4*' '+VETO_CHECKS_DOCSTRING.replace('\n', 12*' '+'\n')
166 def __call__(self, message="", veto_function=None):
167 """Mark this set of files as *PERMANENTLY* ``vetoed`` and hence not
168 meant to be generated.
170 Parameters
171 ----------
172 message : string, optional
173 Optionally provide a descriptive string explaining why this veto
174 occured (and possibly other details) to be saved in the
175 vetofile.
176 veto_function : callable or None, optional
177 Optionally provide the veto check function that caused these files
178 to be vetoed. If provided, that function's string representation
179 and docstring will be logged to the permanent vetofiles to help
180 determine why a veto was triggered.
181 """
182 veto_metadata = {
183 'manifest': format(self.manifest),
184 'veto_uuid': str(uuid.uuid1()),
185 'veto_time': utcnow().isoformat(),
186 'veto_message': format(message),
187 'veto_function': format(getattr(veto_function, '__name__', None)),
188 'veto_function_doc': format(
189 getattr(veto_function, '__doc__', None)
190 ),
191 }
192 LOGGER.debug("Files vetoed: %s", json.dumps(veto_metadata, indent=4))
193 for vfp in self.vetofilepaths:
194 with open(vfp, 'w') as vetofile:
195 json.dump(veto_metadata, vetofile, indent=4, sort_keys=True)
197 def read_json(self):
198 """Read in the JSON dump of this vetofile. Returns ``None`` if the file
199 is not vetoed."""
200 if not self.permanently_vetoed():
201 return None
202 with open(next(self.vetofilepaths)) as vetofile:
203 return json.load(vetofile)
205 def unveto(self):
206 """Remove *PERMANENT* veto status by deleting vetofiles for the file
207 names specified in ``self.manifest`` and event directory specified by
208 ``eventdir``. This does not affect the checks contained in
209 ``self.vetoes`` that are called by ``self.check``, so file generation
210 will still get vetoed if those automated checks fail. This method is
211 more useful for removing permanent vetoes from files that were manually
212 vetoed or vetoed under older pipeline versions."""
213 for vfp in self.vetofilepaths:
214 if os.path.isfile(vfp):
215 os.remove(vfp)
217 @property
218 def vetofilenames(self):
219 """A list of filenames whose existence indicates that these files
220 should *never* be generated (at least for as long as these files
221 exist). The contents of these files can optionally contain text
222 descriptions of why the veto happened."""
223 return ('.{}.veto.json'.format(fname) for fname in self.manifest)
225 @property
226 def vetofilepaths(self):
227 """Absolute paths to vetofiles listed in ``self.vetofilenames``
228 (created by joining ``self.eventdir`` to those filenames)."""
229 return (os.path.join(self.eventdir, vfn) for vfn in self.vetofilenames)
231 def permanently_vetoed(self):
232 """Check whether the files in ``self.manifest`` should be generated
233 based on whether their vetofiles exist (see ``vetofilepaths`` and
234 ``vetofilenames``); as long as this is the case, the files will be
235 considered permanently vetoed and should not be generated."""
236 return any(os.path.isfile(vfp) for vfp in self.vetofilepaths)
238 def check(self):
239 """
240 No return value. Raises a ``VetoException`` if this file has somehow
241 been vetoed, which should be interpreted as meaning that file
242 generation should be aborted.
244 First checks if ``self.permanently_vetoed()`` returns ``True`` (this is
245 to be interpreted as meaning that the file should never be generated);
246 if so, ``VetoException`` is raised.
248 Next, runs all veto functions in ``self.vetoes`` to see if these
249 files have been vetoed in any way. If and when the first veto fails
250 (returns True), its corresponding action (see the ``vetoes`` argument
251 in ``__init__``) is run, further vetoes are not checked, and
252 ``VetoException`` is raised.
254 If these procedures are completed without a ``VetoException`` being
255 raised, the method returns ``None``.
256 """
257 LOGGER.debug("Running veto checks.")
258 if self.flags['VETOED'] == 'true':
259 LOGGER.debug("flagged as VETOED, vetoing %s", self)
260 raise VetoException(("Event flagged as vetoed, so {} "
261 "is vetoed implicitly.").format(self))
262 if self.permanently_vetoed():
263 LOGGER.debug("Permanently vetoed, raising VetoException.")
264 raise VetoException(("Already vetoed: eventdir {} with "
265 "manifest {}").format(self.eventdir,
266 self.manifest))
267 for veto, action in self.vetoes:
268 if veto(self.eventdir):
269 if action is None:
270 LOGGER.debug("Veto %s triggered, permanently vetoing.",
271 veto.__name__ or veto)
272 self("Permanently vetoed.", veto_function=veto)
273 else:
274 LOGGER.debug("Veto %s triggered, taking action: %s",
275 veto.__name__ or veto, action.__name__)
276 action()
277 raise VetoException
280class VetoMixin:
281 """
282 A feature for FileHandler-like objects (producing a manifest and an
283 eventdir) that defines a list of vetoes in a way that inherits additively
284 from superclasses. Notably, provides a ``veto`` property returning a
285 ``VetoHandler`` for this instance. New vetoes can be added to classes by
286 defining them in ``class_vetoes`` (vetoes from superclasses
287 ``class_vetoes`` attributes are dynamically included in the
288 ``VetoHandler`` returned by ``veto``. See ``VetoMixin.vetoes`` docstring
289 for details.
290 """
292 class_vetoes = tuple()
294 @property
295 def veto(self):
296 """Returns a ``VetoHandler`` instance for this ``FileHandler``. See
297 ``VetoHandler`` docstring for more info. Combines vetoes from all
298 superclasses."""
299 return VetoHandler(self.eventdir, self.manifest, self.vetoes())
301 @classmethod
302 def vetoes(cls):
303 r"""The list of veto checks and their corresponding actions called by
304 ``self.veto.checks()``. Returns the vetoes defined in
305 ``cls.class_vetoes``, where vetoes specific to this class are defined,
306 along with the vetoes returned by ``cls.class_vetoes`` in all
307 superclasses; for example, the order of vetoes pulled from
308 class_vetoes (in a class and and its superclasses) for a class F with
309 inheritance tree
311 .. code::
313 A B C
314 \ / /
315 D E
316 \ /
317 F
319 would be: F, D, A, B, E, C (with duplicates after the first instance of
320 a veto check removed, where the ordering of superclasses is listed
321 left-to-right in the above graph).
323 By default, no vetoes are applied to a base FileHandler. Vetoes must be
324 added to FileHandler subclasses. If you really, truly need to get rid
325 of vetoes from superclasses, consider not inheriting from those
326 superclasses or factoring the veto functionality out of those
327 superclasses as a mixin before overriding this property to only return
328 a reduced veto list.
330 A description of the form of this data structure from the
331 ``VetoHandler.__init__`` docstring:
333 """
334 # make sure every veto is a tuple for hashability
335 vetoes = list(tuple(v) for v in cls.class_vetoes)
336 for base in cls.__bases__:
337 if hasattr(base, 'vetoes'):
338 for veto in base.vetoes(): # pylint: disable=no-member
339 veto_tuple = tuple(veto)
340 if veto_tuple not in vetoes:
341 vetoes.append(veto_tuple)
342 return tuple(vetoes)
343 vetoes.__func__.__doc__ += (
344 4*' ' + VETO_CHECKS_DOCSTRING.replace('\n', 12*' '+'\n')
345 )
347 @staticmethod
348 def decorate_checkout(func):
349 """
350 Copy any vetoes for this filehandler over to the generation directory.
351 """
353 @functools.wraps(func)
354 def wrapper(self, *args, **kwargs):
355 """
356 Copy any vetoes for this filehandler over to the generation
357 directory.
358 """
359 tmp_self = func(self, *args, **kwargs)
360 try:
361 for vfile in self.veto.vetofilenames:
362 src = os.path.join(self.eventdir, vfile)
363 dest = os.path.join(tmp_self.eventdir, vfile)
364 if os.path.isfile(src):
365 LOGGER.info('Found vetofile %s, copying to temp directory.',
366 src)
367 shutil.copy(src, dest)
368 os.chmod(dest, 0o400)
369 return tmp_self
370 except: # noqa
371 shutil.rmtree(tmp_self.rundir)
372 raise
374 return wrapper
376 @staticmethod
377 def decorate_checkin(func):
378 """
379 If a ``VetoException`` was raised during generation, copy vetoes from
380 the temporary event directory back to the parent directory and commit
381 them.
382 """
384 @functools.wraps(func)
385 def wrapper(self, gen_result, *args, **kwargs):
386 """
387 If a ``VetoException`` was raised during generation, copy vetoes
388 from the temporary event directory back to the parent directory and
389 commit them. ``gen_result`` here refers to the ``GenerationResult``
390 that is being checked in.
391 """
392 try:
393 return func(self, gen_result, *args, **kwargs)
394 except VetoException as exc:
395 commit_msg = (
396 ("Veto activated on {}: {}.\nCopying vetofiles:\n"
397 "{}\n").format(self, exc,
398 '\n'.join(self.veto.vetofilenames))
399 )
400 LOGGER.info("Veto activated; comitting vetofiles: %s",
401 self.veto.vetofilepaths)
402 generation_dir_copy_vetoes(self, gen_result.fh.eventdir)
403 self.git.commit_changes(commit_msg)
404 raise
406 return wrapper
408 @staticmethod
409 def decorate_generate(func):
410 """
411 Check whether this filehandler has been vetoed before proceeding with
412 generation.
413 """
415 @functools.wraps(func)
416 def wrapper(self, *args, **kwargs):
417 """
418 Check whether this filehandler has been vetoed before proceeding
419 with generation.
420 """
421 try:
422 self.veto.check()
423 except VetoException as err:
424 return GenerationResult(self, err=err)
425 return func(self, *args, **kwargs)
427 return wrapper