Coverage for /opt/anaconda/lib/python3.7/site-packages/llama/serve/gui/__main__.py : 26%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#! (c) Stefan Countryman, 2019
3"""
4Serve pages for events in the current run. Runs a ``Flask`` server in debug
5mode. Specify the domain using environment variable ``LLAMA_DOMAIN`` (default:
6{DEFAULT_LLAMA_DOMAIN}) and the port as ``LLAMA_GUI_PORT`` (default:
7{DEFAULT_LLAMA_GUI_PORT}).
8"""
10# Some useful links for developing:
12# https://stackoverflow.com/questions/50308214/python-3-alternatives-for-getinitargs
13# https://stackoverflow.com/questions/45911705/why-use-os-setsid-in-python
14# http://code.activestate.com/lists/python-list/52627/
15# https://github.com/pallets/werkzeug/issues/1511
16# https://werkzeug.palletsprojects.com/en/0.15.x/middleware/profiler/
17# https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvi-debugging-testing-and-profiling-legacy
18# https://techoverflow.net/2018/03/30/copying-strings-to-the-clipboard-using-pure-javascript/
19# https://uniwebsidad.com/libros/explore-flask/chapter-8/custom-filters
20# https://pypi.org/project/graphviz/
21# https://support.sendwithus.com/jinja/formatting_numbers/
22# https://vecta.io/blog/best-way-to-embed-svg/
23# http://flask.pocoo.org/snippets/8/
24# https://projects.raspberrypi.org/en/projects/python-web-server-with-flask/2
25# http://flask.pocoo.org/snippets/8/
26# https://stackoverflow.com/questions/20212894/how-do-i-get-flask-to-run-on-port-80
28import os
29import logging
30import asyncio
31import json
32import html
33from io import StringIO
34from html.parser import HTMLParser
35from tempfile import TemporaryDirectory
36from multiprocessing import Process, Queue
37from hashlib import sha256
38from pathlib import Path
39from mimetypes import guess_type
40from shutil import rmtree
41from socket import gethostname
42from urllib.parse import urlencode, quote_plus, urlsplit
43from math import ceil
44from functools import wraps
45from time import sleep
46from collections import OrderedDict, namedtuple
47from astropy.io import fits
48from astropy.table import Table
49import psutil
50# try: # handle newer versions of werkzeug, which move this to middleware
51# from werkzeug.middleware.profiler import ProfilerMiddleware
52# except ImportError:
53# from werkzeug.contrib.profiler import ProfilerMiddleware
54import pygments
55from flask import (
56 Flask,
57 render_template,
58 request,
59 Response,
60 abort,
61 send_file,
62 redirect,
63)
64from llama.run import Run, past_runs, SORTKEYS
65from llama.utils import (
66 __version__,
67 LOGDIR,
68 DEFAULT_RUN_DIR,
69 PAST_RUNS_DIR,
70 GenerationError,
71)
72from llama.cli import get_logging_cli, CliParser
73from llama.filehandler import FileGraph, NODELEGEND_COLORS
74from llama.flags import FlagDict, FLAG_PRESETS
75from llama.files.skymap_info.__main__ import run_on_args
76from llama.versioning import GitRepoUninitialized
77from llama.serve.jupyter import running_servers
78from llama.serve.gui.domain import LLAMA_DOMAIN, DEFAULT_LLAMA_DOMAIN
79from llama.serve.gui import (
80 LLAMA_REL_URL_HOME,
81 LLAMA_GUI_PORT,
82 DEFAULT_LLAMA_GUI_PORT,
83 LLAMA_GUI_USERNAME,
84 LLAMA_GUI_PASSWORD,
85 FILE_PREVIEWS,
86 FULL_WIDTH_FILE_PREVIEWS,
87)
88from llama.event import EVENT_AUX_PATHS
90__doc__ = __doc__.format(DEFAULT_LLAMA_GUI_PORT=DEFAULT_LLAMA_GUI_PORT,
91 DEFAULT_LLAMA_DOMAIN=DEFAULT_LLAMA_DOMAIN)
92GUI_LOG = os.path.join(LOGDIR, 'gui.log')
93HOSTNAME = 'gwhen.com'
94SVG_FILENAME = 'files.svg'
95EVENTS_PER_PAGE = 10
96LOGGER = logging.getLogger(__name__)
97WEBCACHE = '.webcache'
98EVENT_AUX_PATHS.append(WEBCACHE)
99_QUEUE_HOLDER = []
100PROC_FILTERS = OrderedDict()
101# PROC_FILTERS["llama listen gcn"] = lambda cmd: ('python' in cmd[0] and
102# any('gcnd' in a for a in cmd))
103# PROC_FILTERS["llama listen lvalert"] = lambda cmd: ('bash' in cmd[0] and
104# any('lvalertd' in a for a in cmd))
105# PROC_FILTERS["llama run"] = lambda cmd: ('python' in cmd[0] and
106# any('gwhend' in a for a in cmd))
107# the following startup commands are run with os.system
108PROC_STARTS = {
109 # "llama listen gcn": "llama listen gcn",
110 # "llama listen lvalert": "llama listen lvalert",
111 # "llama run": "llama run",
112}
113assert all(k in PROC_FILTERS for k in PROC_STARTS)
114DEFAULT_RUN_DIR_NAME = os.path.basename(os.path.abspath(DEFAULT_RUN_DIR))
115PAST_RUNS_DIR_NAME = os.path.basename(os.path.abspath(PAST_RUNS_DIR))
116RELATIVE_RUN_DIRS = (
117 '/' + DEFAULT_RUN_DIR_NAME,
118 '/' + PAST_RUNS_DIR_NAME + '/<run_name>',
119)
120USE_AUTH = [False]
121PLAINTEXT_EXTENSIONS = [
122 'txt',
123 'tex',
124 'json',
125 'xml',
126 'log',
127 'yml',
128]
129try:
130 PYGMENTS_STYLES = pygments.formatters.HtmlFormatter(
131 style=pygments.styles.get_style_by_name('paraiso-dark'),
132 ).get_style_defs('.highlight')
133except pygments.util.ClassNotFound:
134 PYGMENTS_STYLES = ''
136Proc = namedtuple("Proc", ("endpoint", "procs"))
139class AddClassesToTerminalTags(HTMLParser):
140 """
141 Simple HTML parser that adds classes to tags based on tag type. Only adds
142 them to tags with no children.
143 """
145 def __init__(self, tags):
146 """Specify a dictionary of ``tagname: classes_to_add``
147 as ``tags`` where ``classes_to_add`` is a list of strings.
148 These classes will be added to tags of the corresponding type.
149 """
150 super().__init__()
151 self.tags = tags
152 self._last = None
153 self._last_tag = ''
154 self._last_tag_unmodified = ''
155 self._buffer = ''
156 self._original = ''
157 self._result = ''
159 def feed(self, data):
160 self._original = data
161 try:
162 super().feed(data)
163 res = self._result
164 self._result = ''
165 self._original = ''
166 return res
167 except ValueError:
168 res = self._original
169 self._result = ''
170 self._original = ''
171 return res
173 def error(self, message):
174 LOGGER.error("ERROR PARSING HTML: %s", message)
175 raise ValueError(message)
177 @staticmethod
178 def attr_strings(attrs):
179 return ''.join([' '+k if v is None else f" {k}={json.dumps(v)}"
180 for k, v in attrs])
182 def handle_starttag(self, tag, attrs):
183 self._result += self._last_tag_unmodified + self._buffer
184 self._buffer = ''
185 self._last_tag_unmodified = f'<{tag}{self.attr_strings(attrs)}>'
186 if tag in self.tags:
187 for i, attr in enumerate(attrs):
188 if attr[0] == 'class':
189 classes = set(attr[1].split()).union(self.tags[tag])
190 attrs[i] = ' '.join(classes)
191 break
192 else:
193 attrs.append(('class', ' '.join(self.tags[tag])))
194 self._last_tag = f'<{tag}{self.attr_strings(attrs)}>'
196 def handle_endtag(self, tag):
197 self._result += self._last_tag + self._buffer + f'</{tag}>'
198 self._last_tag = ''
199 self._last_tag_unmodified = ''
200 self._buffer = ''
202 def handle_data(self, data):
203 self._buffer += html.escape(data)
205 def handle_startendtag(self, tag, attrs):
206 self._result += self._last_tag_unmodified + self._buffer
207 self._last_tag = ''
208 self._last_tag_unmodified = ''
209 self._buffer = ''
210 self._result += f'<{tag}{self.attr_strings(attrs)} />'
213def print_fits_headers(path):
214 """
215 Read FITS file located at ``path`` and print its fits header info.
216 """
217 buf = StringIO()
218 with fits.open(path) as hdulist:
219 buf.write("FITS Header Summary\n")
220 buf.write("===================\n\n")
221 hdulist.info(output=buf)
222 buf.seek(0)
223 result = buf.read()
224 for i, header in enumerate(hdulist):
225 title = f"FITS Header #{i}"
226 result += '\n\n ' + title + '\n ' + '-'*len(title) + '\n\n'
227 for key, value in header.header.items():
228 result += f' {key:<10} {value}\n'
229 return result
232def flagpreset_js(preset):
233 """Return javascript that can be used as the ``onclick`` script for flag
234 preset buttons. Takes radio buttons from the parent form of ``this`` whose
235 ``name`` matches each flag name and sets them to ``checked`` if their
236 ``value`` matches the preset's value for that flag (unchecking them if they
237 no longer match)."""
238 flagnames = list(preset)
239 values = list(preset.values())
240 desc = json.dumps('Flag Preset description: ' +
241 getattr(preset, 'description', "Current flag settings"))
242 return """
243 var parent = this.parentElement;
244 var desc = parent.getElementsByClassName('flag-preset-description')[0];
245 desc.textContent = """ + desc + """;
246 var inputs = parent.getElementsByTagName('input');
247 var flagnames = """ + str(flagnames) + """;
248 var values = """ + str(values) + """;
249 for (var i=0; i<inputs.length; i++) {
250 if (inputs[i].type == 'radio') {
251 input = inputs[i];
252 for (var j=0; j<flagnames.length; j++) {
253 if (input.name == flagnames[j]) {
254 if (input.value == values[j]) {
255 if (!input.hasAttribute('checked')) {
256 input.setAttribute('checked', '');
257 input.click();
258 }
259 } else {
260 if (input.hasAttribute('checked')) {
261 input.removeAttribute('checked');
262 }
263 }
264 }
265 }
266 }
267 }
268 """
271def jup_server():
272 """Get the parsed URL of a jupyter server. Return ``None`` if it isn't
273 running or the jupyter server file is not found.
274 """
275 servers = running_servers()
276 if servers is None:
277 return None
278 return urlsplit(servers[0][0])
281def get_draftfile(filehandler):
282 """Get a path to a draft file to be edited on Jupyter.
283 """
284 return filehandler.eventdir+'/.'+filehandler.FILENAME+'.DRAFT'
287def relative_home(run_name=None):
288 """Get the relative path to a run based on the name of the run. Defaults
289 to the default run directory.
290 """
291 if run_name is None:
292 return f"{LLAMA_REL_URL_HOME}/{DEFAULT_RUN_DIR_NAME}"
293 return f"{LLAMA_REL_URL_HOME}/{PAST_RUNS_DIR_NAME}/{run_name}"
296def looks_like_plaintext(path):
297 """Check whether a path seems to be a plaintext file by seeing if the first
298 5000 characters can be read as text. Return ``False`` if a
299 ``FileNotFoundError`` or a ``UnicodeDecodeError`` are raised.
300 """
301 try:
302 with open(path, 'r') as infile:
303 infile.read(5000)
304 return True
305 except (FileNotFoundError, UnicodeDecodeError):
306 return False
309def check_auth(username, password):
310 """This function is called to check if a username / password combination is
311 valid."""
312 return username == LLAMA_GUI_USERNAME and password == LLAMA_GUI_PASSWORD
315def authenticate():
316 """Sends a 401 response that enables basic auth"""
317 return Response('Could not verify your access level for that URL.\n'
318 'You have to login with proper credentials', 401,
319 {'WWW-Authenticate': 'Basic realm="Login Required"'})
322def queue_update(queue):
323 """Read ``FileGraph`` instances from a ``Queue`` in a separate process and
324 ``update`` them to avoid blocking on requests. Will call ``update`` on the
325 messages received from ``queue``."""
326 while True:
327 msg = queue.get()
328 print("Got new message on update queue: ", msg)
329 msg.update()
330 print("Finished updating message: ", msg)
333def queue():
334 """Create a new ``Queue`` for the module if it does not exist."""
335 if not _QUEUE_HOLDER:
336 _QUEUE_HOLDER.append(Queue())
337 return _QUEUE_HOLDER[0]
340def requires_auth(func):
341 """Require basic HTTP auth for the Flask development server. This is a
342 kludge until better WSGI routing can be implemented efficiently."""
344 @wraps(func)
345 def decorated(*args, **kwargs):
346 auth = request.authorization
347 if USE_AUTH[0] and (not auth or not check_auth(auth.username,
348 auth.password)):
349 return authenticate()
350 return func(*args, **kwargs)
352 return decorated
355def get_socket_fds():
356 """Get socket file descriptors for the current process."""
357 proc = psutil.Process(os.getpid())
358 return [c.fd for c in proc.connections()]
361app = Flask(__name__)
362LOGGER.debug("App instantiated")
365def run_route(route, **kwargs):
366 """Route requests relative to current and past runs from
367 ``LLAMA_REL_HOME/<relative_run_dir>``.
369 Parameters
370 ----------
371 route : str
372 A ``Flask.route`` route relative to the relevant run directory, e.g.
373 ``/<eventid>/<filename>``.
374 **kwargs
375 Keyword arguments for ``Flask.route``.
376 """
377 LOGGER.debug("Decorator created for %s, %s", route, kwargs)
379 def decorator(func):
380 """Register ``func`` along the specified relative routes.
381 """
382 LOGGER.debug("Decorating %s", func)
383 for reldir in RELATIVE_RUN_DIRS:
384 full_route = LLAMA_REL_URL_HOME+reldir+route
385 LOGGER.debug("Routing %s to %s", full_route, func)
386 app.route(full_route, **kwargs)(func)
387 return func
389 return decorator
392def get_event_from_run(eventid, rundir=None):
393 """Make sure that the ``eventid`` glob expression exists and is unique in
394 this run. It is okay if it is a glob expression just so long as it uniquely
395 specifies an event. Returns the corresponding event object. Aborts with a
396 404 error if an event is not uniquely specified."""
397 kwargs = {} if rundir is None else {'rundir': rundir}
398 events = Run(**kwargs).downselect(eventid_filter=eventid).events
399 if not len(events) == 1:
400 abort(404)
401 return events[0]
404def event_files_graph(event, urls):
405 """Return a visualization of the dependency graph for this event and the
406 current state of the files therein for embedding. ``urls`` is the URL
407 format string to be used with the output SVG images for interactivity."""
408 fhm = event.files
409 return get_graphviz(fhm, event, urls)
412def cachedfilename(event, hashable, ext=''):
413 """Get a cached filename (as an absolute path) for an event based on the
414 current library version combined with the hash of some hashable data (which
415 should be the object that deterministically leads to the creation of the
416 cached output). Optionally specify a file extension to use. The data will
417 be stored in the directory specified by the ``WEBCACHE`` variable within
418 ``event.eventdir`` (the directory will be created if it does not exist when
419 ``cachedfilename`` is called). This leads to automatic cache invalidation
420 for updated code versions and distinguishes between different input
421 arguments."""
422 shasum = sha256(format(hash(hashable)).encode()).hexdigest()
423 outdir = os.path.join(event.eventdir, WEBCACHE)
424 if not os.path.isdir(outdir):
425 os.mkdir(outdir)
426 return os.path.join(outdir, "v-{}-{}{}".format(__version__, shasum, ext))
429def get_graphviz(fhm, event, urls):
430 """Fetch a graphviz svg graph, regenerating it if any of the files involved
431 in its creation have changed in the git history."""
432 regen = False
433 outfilepath = Path(cachedfilename(event, fhm, ext='.svg'))
434 if request.args.get('del-cache', 'false') == 'true':
435 regen = True
436 else:
437 paths = [Path(event.eventdir, f)
438 for fh in fhm.values()
439 for f in [fh.FILENAME] + list(fh.auxiliary_paths)]
440 paths.append(outfilepath)
441 paths.append(Path(event.eventdir, 'FLAGS.json'))
442 t_sorted_files = sorted((p for p in paths if p.exists()),
443 key=lambda p: p.stat().st_mtime)
444 if not t_sorted_files:
445 regen = True
446 elif outfilepath != t_sorted_files[-1]:
447 regen = True
448 else:
449 relevant_times = hashes(event, *paths, pretty='%ct') or [0]
450 if outfilepath.stat().st_mtime < int(relevant_times[0]):
451 regen = True
452 if regen:
453 fhm.dependency_graph(str(outfilepath), urls=urls)
454 with open(outfilepath) as outf:
455 return outf.read()
458def request_hash():
459 """Get a hashable representation of the current request. Will only work for
460 "GET" requests, otherwise a ``ValueError`` is raised. Will ignore
461 ``request.files`` since this should only be used for uploads."""
462 if request.method != 'get':
463 raise ValueError("Can only request hashable get requests.")
464 return (
465 ("authorization", request.authorization),
466 ("args", request.args),
467 ("form", request.form),
468 ("base_url", request.base_url),
469 )
472def serve_file(event, filehandler, run_name=None):
473 """Serve a summary page for this ``filehandler`` class for a specific
474 ``eventid`` and commit (as parsed from the request arguments). If
475 ``download=true`` appears in the query string, download this file instead
476 of returning a summary page."""
477 fh = filehandler(event)
478 if request.args.get("getsvg", 'false') == 'true':
479 return render_template(
480 'graphviz-plot.html',
481 event=event,
482 relhome=relative_home(run_name=run_name),
483 commit=request.args.get('hash'),
484 svg_render=lambda event, urls: filehandler_graph(event,
485 filehandler,
486 urls),
487 )
488 if request.args.get("getfile") == 'true':
489 return send_file(
490 fh.fullpath,
491 mimetype=guess_type(fh.FILENAME)[0],
492 attachment_filename=fh.filename_for_download,
493 as_attachment=False,
494 )
495 if request.args.get("download") == 'true':
496 return send_file(
497 fh.fullpath,
498 mimetype=guess_type(fh.FILENAME)[0],
499 attachment_filename=fh.filename_for_download,
500 as_attachment=True,
501 )
502 return render_template(
503 'file.html',
504 pygments_styles=PYGMENTS_STYLES,
505 run_name=run_name or DEFAULT_RUN_DIR_NAME,
506 version=__version__,
507 domain=LLAMA_DOMAIN,
508 jup_url=jup_server(),
509 processes=get_processes(),
510 hashes=hashes,
511 psutil=psutil,
512 fh=fh,
513 os=os,
514 get_draftfile=get_draftfile,
515 nodelegend=NODELEGEND_COLORS,
516 relhome=relative_home(run_name=run_name),
517 home=LLAMA_REL_URL_HOME,
518 event=event,
519 svg_filename=type(fh).__name__+'.svg',
520 git_path=fh.FILENAME,
521 hostname=gethostname(),
522 svg_render=lambda event, urls: filehandler_graph(event, filehandler,
523 urls),
524 commit=request.args.get('hash'),
525 looks_like_plaintext=looks_like_plaintext,
526 plaintext_extensions=PLAINTEXT_EXTENSIONS,
527 print_fits_headers=print_fits_headers,
528 pygments=pygments,
529 AddClassesToTerminalTags=AddClassesToTerminalTags,
530 )
533def filehandler_graph(event, filehandler, urls):
534 """Generate and serve the dependency graph for this specific
535 filehandler and commit hash. The ``filehandler`` graph name is just the
536 name of the ``filehandler`` with '.svg' appended. ``urls`` is the URL
537 format string to be used with the output SVG images for interactivity."""
538 fh = filehandler(event)
539 return get_graphviz(fh.subgraph, event, urls)
542def hashes(event, *paths, **kwargs):
543 """Safely check for git hashes on this path with ``event.git.hashes``. If a
544 ``ValueError`` is raised, return an empty list. Used for templating where
545 try/except is not feasible."""
546 try:
547 return event.git.hashes(*paths, **kwargs)
548 except (GitRepoUninitialized, ValueError):
549 return []
552@run_route('/<eventid>/<filename>', methods=['GET'])
553@requires_auth
554def file(eventid, filename, run_name=None):
555 """Serve a ``filehandler`` under
556 ``relative_home(run_name)+"/<eventid>/<filename>"`` for the filename
557 defined by this ``FileHandler`` instance. Returns a ``dict`` of registered
558 functions. ``filename`` will be reduced to a basename."""
559 event = get_event_from_run_and_commit(eventid,
560 commit=request.args.get('hash'),
561 run_name=run_name)
562 match = [type(f) for f in event.files.values() if f.FILENAME == filename]
563 if match:
564 return serve_file(event, match[0], run_name=run_name)
565 fullpath = os.path.join(event.eventdir, os.path.basename(filename))
566 # serve other files as downloads
567 if os.path.isfile(fullpath):
568 return send_file(
569 fullpath,
570 mimetype=guess_type(filename)[0],
571 attachment_filename=event.git.filename_for_download(filename),
572 as_attachment=True,
573 )
574 abort(404)
577def get_processes():
578 """Get processes that should be running on the server for the sake of
579 monitoring server status."""
580 processes = {k: list() for k in PROC_FILTERS}
581 for pid in psutil.pids():
582 try:
583 proc = psutil.Process(pid)
584 for procname, test in PROC_FILTERS.items():
585 cmd = proc.cmdline()
586 if cmd and test(cmd):
587 processes[procname].append(proc)
588 except (psutil.AccessDenied, psutil.ZombieProcess,
589 psutil.NoSuchProcess):
590 pass
591 return {k: Proc('/processes/'+k, processes[k]) for k in processes}
594@app.route('/processes/<procname>', methods=['POST'])
595@requires_auth
596def start_proc(procname):
597 """Try to start or kill a process by name. If the process is not one of the
598 processes named in ``PROC_STARTS``, abort with error 400 for a bad
599 request."""
600 if request.form.get("start", "false") == "true":
601 if procname not in PROC_STARTS:
602 abort(400)
603 fds = get_socket_fds()
604 try:
605 pid = os.fork()
606 if pid == 0:
607 # child process; close file descriptors
608 for fd in fds:
609 os.close(fd)
610 os.system(PROC_STARTS[procname])
611 exit(0)
612 else:
613 # parent process, just return the pid and go back to what we
614 # were doing.
615 sleep(0.3)
616 message = quote_plus('Ran '+PROC_STARTS[procname])
617 return redirect(f'/{DEFAULT_RUN_DIR_NAME}/?message={message}')
618 except OSError as err:
619 LOGGER.error("fork #1 failed: %d (%s)", err.errno, err.strerror)
620 raise
621 elif request.form.get("kill", "false") == "true":
622 if procname not in PROC_FILTERS:
623 abort(400)
624 kill_list = list()
625 for pid in psutil.pids():
626 proc = psutil.Process(pid)
627 try:
628 cmd = proc.cmdline()
629 if cmd and PROC_FILTERS[procname](cmd):
630 proc.kill()
631 kill_list.append(pid)
632 except (psutil.AccessDenied, psutil.ZombieProcess,
633 psutil.NoSuchProcess):
634 pass
635 message = quote_plus('Killed '+', '.join(str(p) for p in kill_list))
636 return redirect(f'/{DEFAULT_RUN_DIR_NAME}/?message={message}')
637 abort(400)
640@run_route('/<eventid>/', methods=['POST'])
641@requires_auth
642def update_event(eventid, run_name=None):
643 """Get ``Event.files.update`` running. If ``run_name`` is not None, treat
644 this like a past run."""
645 kwargs = dict()
646 if run_name is not None:
647 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name)
648 event = get_event_from_run(eventid, **kwargs)
649 if request.form.get('CONTAINS_FLAGS', 'false') == 'true':
650 for flagname in FlagDict.DEFAULT_FLAGS:
651 event.flags[flagname] = request.form.get(
652 flagname,
653 FlagDict.DEFAULT_FLAGS[flagname]
654 )
655 msg = ', '.join((k+'='+v for k, v in event.flags.items()))
656 event.git.init()
657 event.git.commit_changes(f"From GUI: updated flags to {msg}")
658 message = 'message='+quote_plus("Set flags: " + msg)
659 return redirect(request.base_url+'?'+message)
660 if request.form.get('generate', 'false') == 'true':
661 queue().put(event.files)
662 message = 'message='+quote_plus('Started generating files for' +
663 event.eventid)
664 return redirect(request.base_url+'?'+message)
665 abort(400)
668@run_route('/<eventid>/<filename>', methods=['POST'])
669@requires_auth
670def upload_file(eventid, filename, run_name=None):
671 """Manually upload a file and commit it to an event's history, overwriting
672 the previous version. Can only upload to the latest version of the file. If
673 no file is sent but 'lock' is 'true' or 'false', either lock or unlock the
674 corresponding file."""
675 kwargs = dict()
676 if run_name is not None:
677 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name)
678 event = get_event_from_run(eventid, **kwargs)
679 match = [f for f in event.files.values() if f.FILENAME == filename]
680 if not match:
681 abort(404)
682 fh = match[0]
683 if 'file' not in request.files:
684 # try deleting the file
685 if request.form.get('delete', 'false') == 'true':
686 fh.delete()
687 message = 'message='+quote_plus('Successfully deleted '+filename)
688 return redirect(request.base_url+'?'+message)
689 # try generating the file
690 if request.form.get('generate', 'false') == 'true':
691 queue().put(fh.subgraph)
692 # wait a moment so that the intent marker can be placed and
693 # recognized in the updated page
694 sleep(1)
695 message = 'message='+quote_plus('Started generating '+filename)
696 return redirect(request.base_url+'?'+message)
697 # try creating a new file and editing it on Jupyter
698 if 'edit_new_text_file' in request.form:
699 jup_url = jup_server()
700 draftfile = get_draftfile(fh)
701 if not os.path.isfile(draftfile):
702 with open(draftfile, 'w') as outfile:
703 outfile.write(
704 "Write thie file's contents and then save it and\n"
705 "rename it to:\n\n"
706 f" {fh.FILENAME}\n\n"
707 "Do NOT rename it before you've finished it; the\n"
708 "LLAMA update daemon will see the existing file and\n"
709 "assume that it is ready to use.\n"
710 )
711 return redirect(f"{jup_url.scheme}://{jup_url.netloc}/edit/"
712 f"{run_name or DEFAULT_RUN_DIR_NAME}/"
713 f"{event.eventid}/{os.path.basename(draftfile)}?"
714 f"{jup_url.query}")
715 # try locking/unlocking the file
716 if 'veto' in request.form:
717 if request.form['veto'] == 'true':
718 fh.veto(message="Manual veto from GUI.")
719 msg = f'Permanently vetoed {fh.FILENAME}'
720 elif request.form['veto'] == 'false':
721 fh.veto.unveto()
722 msg = (f'Removed permanent veto on {fh.FILENAME} (might be '
723 'automatically reapplied)')
724 else:
725 abort(400)
726 fh.git.init()
727 fh.git.commit_changes(f"From GUI: {msg}")
728 return redirect(request.base_url+'?message='+quote_plus(msg))
729 if 'remove_intent' in request.form:
730 if request.form['remove_intent'] != 'true':
731 abort(400)
732 fh.intent.delete()
733 change = f'Deleted intent file for {fh.FILENAME}'
734 msg = (f"{change}; in-progress file generation will not "
735 "be committed. You are now free to regenerate this file.")
736 fh.git.init()
737 try:
738 fh.git.commit_changes(f"From GUI: {change}")
739 except GenerationError:
740 pass
741 return redirect(request.base_url+'?message='+quote_plus(msg))
742 if 'remove_cooldown' in request.form:
743 if request.form['remove_cooldown'] != 'true':
744 abort(400)
745 fh.cooldown.delete()
746 msg = "Deleted cooldown file (consider deleting intent file too)"
747 fh.git.init()
748 try:
749 fh.git.commit_changes(f"From GUI: {msg}")
750 except GenerationError:
751 pass
752 return redirect(request.base_url+'?message='+quote_plus(msg))
753 if 'lock' in request.form:
754 if request.form['lock'] == 'true':
755 fh.lock.lock()
756 change = 'Locked'
757 elif request.form['lock'] == 'false':
758 fh.lock.unlock()
759 change = 'Unlocked'
760 else:
761 abort(400)
762 fh.git.init()
763 fh.git.commit_changes(
764 "Manually {} {} from browser.".format(
765 change.lower(),
766 fh.FILENAME,
767 )
768 )
769 message = 'message='+quote_plus(change+' file.')
770 return redirect(request.base_url+'?'+message)
771 upload = request.files['file']
772 if upload.FILENAME == '':
773 # abort(418) # tfw when u r a t pot
774 message = 'message='+quote_plus('No file selected, upload failed.')
775 return redirect(request.base_url + '?' +
776 request.query_string.decode() + message)
777 if upload:
778 upload.save(fh.fullpath)
779 fh.git.init()
780 fh.git.commit_changes(("Manually uploaded {} from "
781 "browser.").format(upload.FILENAME))
782 message = 'message='+quote_plus('Upload successful.')
783 return redirect(request.base_url+'?'+message)
786@run_route('/', methods=['POST'])
787@requires_auth
788def new_event(run_name=None):
789 """Create a new event."""
790 args = list()
791 if run_name is not None:
792 args += ['--outdir', os.path.join(PAST_RUNS_DIR, run_name,
793 request.form['graceid'])]
794 if 'graceid' not in request.form:
795 abort(400)
796 args += ['--graceid', request.form['graceid']]
797 if request.form.get('skymap'):
798 args += ['--skymap', request.form['skymap']]
799 if request.form.get('overwrite') == 'true':
800 args.append('--clobber')
801 args.append('--flags')
802 args += [f+'='+request.form.get(f, FlagDict.DEFAULT_FLAGS[f])
803 for f in FlagDict.DEFAULT_FLAGS]
804 try:
805 run_on_args(args, error_on_failure=True)
806 url = (relative_home(run_name=run_name) + '/' +
807 request.form['graceid'])
808 except RuntimeError as err:
809 msg = quote_plus(f'Error getting skymap with args {args}: {err}')
810 url = f'?message={msg}'
811 return redirect(url)
814def event_version_checkout_rundir(eventid, commit, run_name=None):
815 """Return a temporary (i.e. deletable) run directory where an
816 ``Event.clone`` of a specific ``eventid`` ``commit`` can be cached for
817 serving/manipulating in a consistent way while also allowing for cache
818 flushing. Will not create any intermediate or final directories."""
819 kwargs = dict()
820 if run_name is not None:
821 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name)
822 event = get_event_from_run(eventid, **kwargs)
823 return os.path.join(event.eventdir, WEBCACHE, 'VERSION-CHECKOUT', commit)
826def get_event_from_run_and_commit(eventid, commit=None, run_name=None):
827 """Same as ``get_event_from_run``, but if a ``commit`` is specified, will
828 clone the event at the specified ``commit`` in a repeatable (read:
829 cacheable) way so that that snapshot can be used in the future (but also
830 easily flushed)."""
831 kwargs = dict()
832 if run_name is not None:
833 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name)
834 event = get_event_from_run(eventid, **kwargs)
835 if commit is not None:
836 temp_rundir = event_version_checkout_rundir(eventid, commit=commit,
837 run_name=run_name)
838 if not os.path.isdir(temp_rundir):
839 os.makedirs(temp_rundir)
840 event = event.clone(commit=commit, rundir=temp_rundir, clobber=True)
841 # async_rm(event.eventdir, delay=7)
842 return event
845@run_route('/', methods=['GET'])
846@requires_auth
847def index(run_name=None):
848 """Return a homepage for the current run."""
849 args = {
850 "reverse": request.args.get('reverse', 'True') == 'True',
851 "eventid_filter": request.args.get("eventid_filter", "S*"),
852 }
853 run_kwargs = dict()
854 if run_name is not None:
855 run_kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name)
856 events = Run(**run_kwargs).downselect(sortkey=SORTKEYS['lvk'],
857 **args).events
858 page = int(request.args.get('page', 0))
859 start = EVENTS_PER_PAGE*page
860 end = EVENTS_PER_PAGE*(page+1)
861 downselected_events = events[start:end]
862 pages = ceil(len(events)/float(EVENTS_PER_PAGE))
863 return render_template(
864 'index.html',
865 pygments_styles=PYGMENTS_STYLES,
866 run_name=run_name or DEFAULT_RUN_DIR_NAME,
867 version=__version__,
868 domain=LLAMA_DOMAIN,
869 jup_url=jup_server(),
870 processes=get_processes(),
871 hashes=hashes,
872 psutil=psutil,
873 nodelegend=NODELEGEND_COLORS,
874 relhome=relative_home(run_name=run_name),
875 home=LLAMA_REL_URL_HOME,
876 svg_render=event_files_graph,
877 downselected_events=downselected_events,
878 start=start+1,
879 end=min(end, len(events)),
880 hostname=gethostname(),
881 git_path='--', # include all paths in run histories
882 total_events=len(events),
883 pages=pages,
884 page=page,
885 next_page='?'+urlencode(args)+'&page='+str(page+1) if page+1 < pages else "",
886 prev_page='?'+urlencode(args)+'&page='+str(page-1) if page > 0 else "",
887 FlagDict=FlagDict,
888 getattr=getattr,
889 flagpreset_js=flagpreset_js,
890 starting_flags=FlagDict.DEFAULT_FLAGS,
891 starting_flag_desc=FlagDict.DEFAULT_FLAGS.description,
892 )
895async def async_rm(path, delay=5):
896 """Asynchronously wait for ``delay`` seconds before deleting ``path``
897 recursively using ``os.unlink`` for files and ``shutil.rmtree`` for
898 directories. Ignores any ``IOError`` on the assumption that these arise
899 from harmless tempfile deletion race conditions. Use this to clean up
900 tempfiles."""
901 await asyncio.sleep(delay)
902 try:
903 if os.path.isdir(path):
904 rmtree(path)
905 else:
906 os.unlink(path)
907 except IOError as err:
908 LOGGER.warning("IOError during scheduled removal of %s: %s", path, err)
911@run_route('/<eventid>/')
912@requires_auth
913def event_page(eventid, run_name=None):
914 """Return the event page for a given event."""
915 event = get_event_from_run_and_commit(eventid,
916 commit=request.args.get('hash'),
917 run_name=run_name)
918 if request.args.get("getsvg", 'false') == 'true':
919 return render_template(
920 'graphviz-plot.html',
921 event=event,
922 relhome=relative_home(run_name=run_name),
923 commit=request.args.get('hash'),
924 svg_render=event_files_graph,
925 )
926 if request.args.get("download", 'false') == 'true':
927 with TemporaryDirectory() as tmpdir:
928 tgz = os.path.join(tmpdir, event.eventid+'.tar.gz')
929 event.save_tarball(tgz)
930 return send_file(
931 tgz,
932 mimetype=guess_type(tgz)[0],
933 attachment_filename=event.eventid+'.tar.gz',
934 as_attachment=True,
935 )
936 return render_template(
937 'event.html',
938 pygments_styles=PYGMENTS_STYLES,
939 run_name=run_name or DEFAULT_RUN_DIR_NAME,
940 file_previews=[getattr(event.files, f) for f in FILE_PREVIEWS],
941 full_width_file_previews=[getattr(event.files, f)
942 for f in FULL_WIDTH_FILE_PREVIEWS],
943 version=__version__,
944 domain=LLAMA_DOMAIN,
945 jup_url=jup_server(),
946 processes=get_processes(),
947 hashes=hashes,
948 psutil=psutil,
949 event=event,
950 nodelegend=NODELEGEND_COLORS,
951 relhome=relative_home(run_name=run_name),
952 home=LLAMA_REL_URL_HOME,
953 svg_render=event_files_graph,
954 git_path='--', # include all paths in run histories
955 commit=request.args.get('hash'),
956 hostname=gethostname(),
957 plaintext_extensions=PLAINTEXT_EXTENSIONS,
958 pygments=pygments,
959 print_fits_headers=print_fits_headers,
960 getattr=getattr,
961 flagpreset_js=flagpreset_js,
962 FlagDict=FlagDict,
963 starting_flags={f: event.flags[f] for f in FlagDict.ALLOWED_VALUES},
964 starting_flag_desc="Current flag settings",
965 AddClassesToTerminalTags=AddClassesToTerminalTags,
966 )
969@app.route('/')
970@requires_auth
971def home():
972 """Get a home page showing available LLAMA analysis runs."""
973 past = {os.path.basename(k): v for k, v in past_runs().items()}
974 return render_template(
975 'home.html',
976 pygments_styles=PYGMENTS_STYLES,
977 run_name=DEFAULT_RUN_DIR_NAME,
978 version=__version__,
979 relhome=relative_home(None),
980 home=LLAMA_REL_URL_HOME,
981 domain=LLAMA_DOMAIN,
982 jup_url=jup_server(),
983 past_runs=past,
984 past_runs_name=PAST_RUNS_DIR_NAME,
985 processes=get_processes(),
986 psutil=psutil,
987 nodelegend=NODELEGEND_COLORS,
988 )
991@app.route(LLAMA_REL_URL_HOME+'/static/style.css')
992@requires_auth
993def css():
994 """Get the CSS file for this app."""
995 return send_file(
996 os.path.join(
997 os.path.dirname(__file__),
998 'static',
999 'style.css',
1000 )
1001 )
1004# these files do not require authorization. they are just favicons.
1005@app.route('/static/<filename>')
1006def rootdir_file(filename):
1007 """Get files in '/static/root' to serve files that should live at root
1008 (like favicons)."""
1009 filename = os.path.basename(os.path.abspath(filename))
1010 dirpath = os.path.join(
1011 os.path.dirname(__file__),
1012 'static',
1013 'root',
1014 )
1015 if not filename in os.listdir(dirpath):
1016 abort(404)
1017 return send_file(os.path.join(dirpath, filename))
1020def get_parser():
1021 """Get an argument parser."""
1022 parser = CliParser(description=__doc__,
1023 parents=(get_logging_cli(GUI_LOG, 'info'),))
1024 parser.add_argument("--use-auth", action="store_true", help="""
1025 Force the user to log in using basic HTTP auth. Requires that
1026 environmental variables ``LLAMA_GUI_USERNAME`` and
1027 ``LLAMA_GUI_PASSWORD`` are both set.""")
1028 return parser
1031def main():
1032 """Run the server."""
1033 parser = get_parser()
1034 args = parser.parse_args()
1035 USE_AUTH[0] = args.use_auth
1036 if args.use_auth:
1037 if LLAMA_GUI_USERNAME is None or LLAMA_GUI_PASSWORD is None:
1038 parser.error("If `--use-auth` is provided, must set username and "
1039 "password using environmental variables "
1040 "`LLAMA_GUI_USERNAME` and `LLAMA_GUI_PASSWORD`")
1041 # app.config['PROFILE'] = True
1042 # app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=os.getcwd(),
1043 # filename_format="{method}.{path}.{time:f}.{elapsed:06f}ms.prof",
1044 # restrictions=[30])
1045 # start background process to handle file graph updates
1046 updater_proc = Process(target=queue_update, args=(queue(),))
1047 updater_proc.daemon = True
1048 updater_proc.start()
1049 print("Started updater queue process with pid ", updater_proc.pid)
1050 # if args.debug:
1051 # app.run(debug=args.debug, port=args.port)
1052 # else:
1053 # from waitress import serve
1054 # serve(app, listen=f'*:{args.port}')
1055 from waitress import serve
1056 serve(app, listen=f'*:{LLAMA_GUI_PORT}')
1057 updater_proc.join()
1060if __name__ == '__main__':
1061 main()