Coverage for /opt/anaconda/lib/python3.7/site-packages/llama/files/gracedb.py : 47%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# (c) Stefan Countryman, 2017
3"""
4FileHandlers for uploading and downloading files from GraceDB plus utilities
5for interacting with GraceDB.
6"""
8import logging
9from abc import abstractproperty
10import datetime
11import shutil
12import tempfile
13import json
14from llama.filehandler import (
15 GenerationError,
16 FileHandler,
17 JSONFile,
18)
19from llama.filehandler.mixins import (
20 OnlineVetoMixin,
21)
22from llama.com.utils import UploadReceipt
23from llama.files.skymap_info import SkymapInfo
24from llama.com.gracedb import GraceDb, HTTPError
25from llama.files.slack import SlackReceiptLlama
27LOGGER = logging.getLogger(__name__)
30class GraceDBReceipt(UploadReceipt):
31 """A log file created when a file is uploaded to GraceDB."""
33 FILENAME_FMT = 'rct_gdb_{}.log'
34 CLASSNAME_FMT = "RctGdb{}"
36 @classmethod
37 def set_class_attributes(cls, subclass):
38 """See ``UploadReceipt.set_class_attributes``; this method additionally
39 adds ``SkymapInfo`` to the ``DEPENDENCIES`` list for ``subclass`` (if
40 it is not implicitly there by virtue of being the uploaded file)."""
41 super().set_class_attributes(subclass)
42 if subclass.UPLOAD is not SkymapInfo:
43 subclass.DEPENDENCIES = tuple(list(subclass.DEPENDENCIES) +
44 [SkymapInfo])
45 # reset class attributes using new values
46 super().set_class_attributes(subclass)
47 return subclass
49 @abstractproperty
50 def log_message(self):
51 """A message to accompany this file on the GraceDB log entry for this
52 event. The last line of this log file is a JSON object describing the
53 result of the upload operation."""
55 @property
56 def gracedb_tags(self):
57 """Specify tags (which affect display and access permissions) for
58 the uploaded GraceDB file. Can be overridden for files that don't
59 fit the default topics:
61 [u'lvem', u'em_follow', u'sky_loc']
63 """
64 return ['lvem', # allows LVEM members to view this file
65 'em_follow', # displays file under EM Followup section
66 'sky_loc'] # displays file under Sky Localization section
68 def _generate(self): # pylint: disable=arguments-differ
69 # perform and log the upload to a tempfile, then rename when finished
70 # so that logfile existence is synonymous with upload success
71 # pylint: disable=invalid-name
72 with tempfile.NamedTemporaryFile(mode='w', dir=self.eventdir,
73 delete=False) as f:
74 # don't generate these as part of the test pipeline!
75 if 'test' in self.eventid:
76 f.write('This seems to be a test event. Aborting upload.\n')
77 else:
78 f.write('Message to be uploaded to GraceDB:\n')
79 f.write('{}\n'.format(self.log_message))
80 # timestamp
81 f.write('Uploading at {}\n'.format(datetime.datetime.now()))
82 # this tagname tells GraceDb that this is analysis performed by
83 # humans
84 res = GraceDb().writeLog(SkymapInfo(self).graceid,
85 self.log_message,
86 self.UPLOAD(self).fullpath,
87 tagname=self.gracedb_tags)
88 if res.status != 201:
89 fmt = 'GraceDb upload failed. Status: {}'
90 msg = fmt.format(res.status)
91 raise GenerationError(msg)
92 rdict = res.json()
93 rjson = json.dumps(rdict)
94 f.write('Finished at {}\n'.format(datetime.datetime.now()))
95 # The last line must be the response JSON. This way other
96 # FileHandlers can read in the response information from this
97 # log.
98 f.write('Response json:\n{}\n'.format(rjson))
99 fname = f.name
100 shutil.move(fname, self.fullpath)
102 # pylint: disable=arguments-differ
103 @classmethod
104 def decorator_dict(cls, upload, log_message="Upload from LLAMA pipeline."):
105 """See ``UploadReceipt.upload_this`` and
106 ``UploadReceipt.decorator_dict``.
108 Parameters
109 ----------
110 upload
111 The decorated ``FileHandler`` class that is being registered for
112 upload.
113 log_message : function or str, optional
114 Either a string whose ``format`` function will be called with the
115 new ``GraceDBReceipt`` subclass as its ``self`` keyword argument or
116 a function taking the new ``GraceDBReceipt`` as its only argument
117 (will become the ``log_message`` property for the new
118 ``GraceDBReceipt``).
120 Returns
121 -------
122 newclassdict : dict
123 A dictionary that can be passed to ``type`` to specify the
124 attributes of a new class.
126 Raises
127 ------
128 TypeError
129 If ``log_message`` is not a formattable string or callable object.
130 """
131 if hasattr(log_message, 'format'):
133 def log(self):
134 return log_message.format(self)
136 elif callable(log_message):
137 log = log_message
138 else:
139 raise TypeError(f"Invalid log_message specified: {log_message}")
141 @property
142 def log_message(self): # pylint: disable=function-redefined
143 return log(self)
145 log_message.__doc__ = f"GraceDB upload log message for {upload}"
146 return {"log_message": log_message}
148 @property
149 def upload_dict(self):
150 """When uploading to GraceDB via the REST API, the GraceDB client
151 produces a dict with useful information about the uploaded file. This
152 method returns that dict. An example instance of this dict:
154 {u'N': 46,
155 u'comment': u'Testing json upload ONE MORE TIME!',
156 u'created': u'2017-03-19T04:49:43.515566+00:00',
157 u'file': (u'https://gracedb.ligo.org/api/events/M278200/'
158 u'files/icecube_neutrino_list.json,4'),
159 u'file_version': 4,
160 u'filename': u'icecube_neutrino_list.json',
161 u'issuer': {u'display_name': u'Stefan Countryman',
162 u'username': u'stefan.countryman@LIGO.ORG'},
163 u'self': u'https://gracedb.ligo.org/api/events/M278200/log/46',
164 u'tag_names': [u'em_follow'],
165 u'tags': u'https://gracedb.ligo.org/api/events/M278200/log/46/tag/'}
167 If the log file doesn't exist or the upload dict cannot be read, this
168 function returns None.
169 """
170 # The upload dict is converted to a single line of JSON and is written
171 # on the last line of the upload receipt log file. This method reads
172 # from that logfile.
173 try:
174 with self.open() as infile:
175 return json.loads(infile.read().strip('\n').split('\n')[-1])
176 except (IOError, ValueError, IndexError):
177 return None
180class GraceDBPollingFileHandler(FileHandler, OnlineVetoMixin):
181 """
182 A data file that can be downloaded from GraceDB as soon as it is ready.
183 SkymapInfo is necessarily a dependency for all subclasses (since we need to
184 extract the GraceID from it). In most cases, this should be the only
185 dependency.
186 """
188 @abstractproperty
189 def remote_filename(self):
190 """The name of the required data in a format recognizable by the remote
191 resource's API. This is quite possibly dynamically generated, and so
192 it must be considered separate from the file handler's name for this
193 file."""
195 DEPENDENCIES = (SkymapInfo,)
197 def _generate(self): # pylint: disable=arguments-differ
198 try:
199 res = GraceDb().files(SkymapInfo(self).graceid, self.remote_filename)
200 except HTTPError as err:
201 raise GenerationError(err.message)
202 with open(self.fullpath, 'w') as outfile:
203 outfile.write(res.read().decode())
206@GraceDBPollingFileHandler.set_class_attributes
207class PAstro(GraceDBPollingFileHandler, JSONFile):
208 """A classification of an event's possible sources by probability."""
210 FILENAME = "p_astro.json"
212 @property
213 def remote_filename(self):
214 return "p_astro.json"
216 @property
217 def p_bbh(self):
218 """The probability that this was a BBH (binary black hole) merger."""
219 return self.read_json()['BBH']
221 @property
222 def p_bns(self):
223 """The probability that this was a BNS (binary neutron star) merger."""
224 return self.read_json()['BNS']
226 @property
227 def p_nsbh(self):
228 """The probability that this was a NSBH (neutron star/black hole)
229 merger."""
230 return self.read_json()['NSBH']
232 @property
233 def p_mass_gap(self):
234 """The probability that at least one of the compact objects in the
235 merger was in the ambiguous mass gap between definite BH and definite
236 NS."""
237 return self.read_json()['MassGap']
239 @property
240 def p_terr(self):
241 """The probability that this event was terrestrial noise."""
242 return self.read_json()['Terrestrial']
244 @property
245 def most_likely_population(self):
246 """Get the population that this event most likely belongs too by seeing
247 whether it is more likely than not to contain NS matter. More
248 precisely, compares ``p_bbh`` to ``p_bns + p_nsbh + p_mass_gap``,
249 returning ``'bbh'`` if the former is larger and ``'bns'`` if the latter
250 is larger. ``p_terr`` is ignored in this calculation."""
251 if self.p_bbh > self.p_bns + self.p_nsbh + self.p_mass_gap:
252 return 'bbh'
253 return 'bns'
256@SlackReceiptLlama.upload_this()
257@JSONFile.set_class_attributes
258class LVCGraceDbEventData(JSONFile, OnlineVetoMixin):
259 """
260 The event dictionary as returned by the GraceDb API. Includes richer
261 information on the single-detector and multi-detector inspiral parameters
262 than is provided in LVAlerts or GCN Notices. Convenience methods are
263 provided for certain commonly-used reconstructed parameters, but full data
264 can always be accessed by running ``read_json``.
265 """
267 FILENAME = 'lvc_gracedb_event_data.json'
268 DEPENDENCIES = (SkymapInfo,)
270 # properties that must bei in the response from GraceDB, or else it will be
271 # rejected; provide a tuple of tuples of keys that must successively map to
272 # objects in the event dictionary.
273 _required_properties = (
274 ('offline',),
275 ('instruments',),
276 )
278 # the current list of instruments that can participate in GW searches
279 GW_DETECTORS = {
280 "H1",
281 "L1",
282 "V1",
283 }
285 def _generate(self):
286 client = GraceDb()
287 skyinfo = SkymapInfo(self)
288 try:
289 if (skyinfo.graceid.startswith("S") or
290 skyinfo.graceid.startswith("MS")):
291 sup = client.superevent(skyinfo.graceid).json()
292 graceid = sup['preferred_event']
293 else:
294 graceid = skyinfo.graceid
295 res = client.event(graceid)
296 except HTTPError as err:
297 raise GenerationError(err.message)
298 event = res.json()
299 for path in self._required_properties:
300 partial_path = event
301 for key in path:
302 try:
303 partial_path = partial_path[key]
304 except KeyError:
305 msg_fmt = ("Could not parse GraceDB event info, missing "
306 "property: event[{}]")
307 msg = msg_fmt.format("][".join(repr(k) for k in path))
308 raise GenerationError(msg)
309 self._write_json(event)
311 @property
312 def snr(self):
313 """The coincident signal-to-noise ratio (SNR). Raises a ``KeyError`` if
314 it is not defined."""
315 return self.read_json()['extra_attributes']['CoincInspiral']['snr']
317 @property
318 def total_mass(self):
319 """The total binary mass. Raises a ``KeyError`` if it is not
320 defined."""
321 event = self.read_json()
322 return event['extra_attributes']['CoincInspiral']['mass']
324 @property
325 def chirp_mass(self):
326 """The chirp mass. Raises a ``KeyError`` if it is not
327 defined."""
328 event = self.read_json()
329 return event['extra_attributes']['CoincInspiral']['mchirp']
331 @property
332 def instruments(self):
333 """A sorted list of the interferometers that participated in creating
334 this trigger, e.g. ``['H1', 'L1']``. Raises a ``ValueError`` if
335 unexpected instruments are found. Raises a ``KeyError`` if it is not
336 defined."""
337 instruments = sorted(self.read_json()['instruments'].split(','))
338 if not self.GW_DETECTORS.issuperset(instruments):
339 raise ValueError(("Got unexpected instruments for {}: "
340 "{}").format(self.eventid, instruments))
341 return instruments
343 @property
344 def offline(self):
345 """Whether this trigger was generated offline."""
346 event = self.read_json()
347 return event['offline']
350assert all(type(p) == tuple for p in LVCGraceDbEventData._required_properties)