Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#! (c) Stefan Countryman, 2019 

2 

3""" 

4Serve pages for events in the current run. Runs a ``Flask`` server in debug 

5mode. Specify the domain using environment variable ``LLAMA_DOMAIN`` (default: 

6{DEFAULT_LLAMA_DOMAIN}) and the port as ``LLAMA_GUI_PORT`` (default: 

7{DEFAULT_LLAMA_GUI_PORT}). 

8""" 

9 

10# Some useful links for developing: 

11 

12# https://stackoverflow.com/questions/50308214/python-3-alternatives-for-getinitargs 

13# https://stackoverflow.com/questions/45911705/why-use-os-setsid-in-python 

14# http://code.activestate.com/lists/python-list/52627/ 

15# https://github.com/pallets/werkzeug/issues/1511 

16# https://werkzeug.palletsprojects.com/en/0.15.x/middleware/profiler/ 

17# https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvi-debugging-testing-and-profiling-legacy 

18# https://techoverflow.net/2018/03/30/copying-strings-to-the-clipboard-using-pure-javascript/ 

19# https://uniwebsidad.com/libros/explore-flask/chapter-8/custom-filters 

20# https://pypi.org/project/graphviz/ 

21# https://support.sendwithus.com/jinja/formatting_numbers/ 

22# https://vecta.io/blog/best-way-to-embed-svg/ 

23# http://flask.pocoo.org/snippets/8/ 

24# https://projects.raspberrypi.org/en/projects/python-web-server-with-flask/2 

25# http://flask.pocoo.org/snippets/8/ 

26# https://stackoverflow.com/questions/20212894/how-do-i-get-flask-to-run-on-port-80 

27 

28import os 

29import logging 

30import asyncio 

31import json 

32import html 

33from io import StringIO 

34from html.parser import HTMLParser 

35from tempfile import TemporaryDirectory 

36from multiprocessing import Process, Queue 

37from hashlib import sha256 

38from pathlib import Path 

39from mimetypes import guess_type 

40from shutil import rmtree 

41from socket import gethostname 

42from urllib.parse import urlencode, quote_plus, urlsplit 

43from math import ceil 

44from functools import wraps 

45from time import sleep 

46from collections import OrderedDict, namedtuple 

47from astropy.io import fits 

48from astropy.table import Table 

49import psutil 

50# try: # handle newer versions of werkzeug, which move this to middleware 

51# from werkzeug.middleware.profiler import ProfilerMiddleware 

52# except ImportError: 

53# from werkzeug.contrib.profiler import ProfilerMiddleware 

54import pygments 

55from flask import ( 

56 Flask, 

57 render_template, 

58 request, 

59 Response, 

60 abort, 

61 send_file, 

62 redirect, 

63) 

64from llama.run import Run, past_runs, SORTKEYS 

65from llama.utils import ( 

66 __version__, 

67 LOGDIR, 

68 DEFAULT_RUN_DIR, 

69 PAST_RUNS_DIR, 

70 GenerationError, 

71) 

72from llama.cli import get_logging_cli, CliParser 

73from llama.filehandler import FileGraph, NODELEGEND_COLORS 

74from llama.flags import FlagDict, FLAG_PRESETS 

75from llama.files.skymap_info.__main__ import run_on_args 

76from llama.versioning import GitRepoUninitialized 

77from llama.serve.jupyter import running_servers 

78from llama.serve.gui.domain import LLAMA_DOMAIN, DEFAULT_LLAMA_DOMAIN 

79from llama.serve.gui import ( 

80 LLAMA_REL_URL_HOME, 

81 LLAMA_GUI_PORT, 

82 DEFAULT_LLAMA_GUI_PORT, 

83 LLAMA_GUI_USERNAME, 

84 LLAMA_GUI_PASSWORD, 

85 FILE_PREVIEWS, 

86 FULL_WIDTH_FILE_PREVIEWS, 

87) 

88from llama.event import EVENT_AUX_PATHS 

89 

90__doc__ = __doc__.format(DEFAULT_LLAMA_GUI_PORT=DEFAULT_LLAMA_GUI_PORT, 

91 DEFAULT_LLAMA_DOMAIN=DEFAULT_LLAMA_DOMAIN) 

92GUI_LOG = os.path.join(LOGDIR, 'gui.log') 

93HOSTNAME = 'gwhen.com' 

94SVG_FILENAME = 'files.svg' 

95EVENTS_PER_PAGE = 10 

96LOGGER = logging.getLogger(__name__) 

97WEBCACHE = '.webcache' 

98EVENT_AUX_PATHS.append(WEBCACHE) 

99_QUEUE_HOLDER = [] 

100PROC_FILTERS = OrderedDict() 

101# PROC_FILTERS["llama listen gcn"] = lambda cmd: ('python' in cmd[0] and 

102# any('gcnd' in a for a in cmd)) 

103# PROC_FILTERS["llama listen lvalert"] = lambda cmd: ('bash' in cmd[0] and 

104# any('lvalertd' in a for a in cmd)) 

105# PROC_FILTERS["llama run"] = lambda cmd: ('python' in cmd[0] and 

106# any('gwhend' in a for a in cmd)) 

107# the following startup commands are run with os.system 

108PROC_STARTS = { 

109 # "llama listen gcn": "llama listen gcn", 

110 # "llama listen lvalert": "llama listen lvalert", 

111 # "llama run": "llama run", 

112} 

113assert all(k in PROC_FILTERS for k in PROC_STARTS) 

114DEFAULT_RUN_DIR_NAME = os.path.basename(os.path.abspath(DEFAULT_RUN_DIR)) 

115PAST_RUNS_DIR_NAME = os.path.basename(os.path.abspath(PAST_RUNS_DIR)) 

116RELATIVE_RUN_DIRS = ( 

117 '/' + DEFAULT_RUN_DIR_NAME, 

118 '/' + PAST_RUNS_DIR_NAME + '/<run_name>', 

119) 

120USE_AUTH = [False] 

121PLAINTEXT_EXTENSIONS = [ 

122 'txt', 

123 'tex', 

124 'json', 

125 'xml', 

126 'log', 

127 'yml', 

128] 

129try: 

130 PYGMENTS_STYLES = pygments.formatters.HtmlFormatter( 

131 style=pygments.styles.get_style_by_name('paraiso-dark'), 

132 ).get_style_defs('.highlight') 

133except pygments.util.ClassNotFound: 

134 PYGMENTS_STYLES = '' 

135 

136Proc = namedtuple("Proc", ("endpoint", "procs")) 

137 

138 

139class AddClassesToTerminalTags(HTMLParser): 

140 """ 

141 Simple HTML parser that adds classes to tags based on tag type. Only adds 

142 them to tags with no children. 

143 """ 

144 

145 def __init__(self, tags): 

146 """Specify a dictionary of ``tagname: classes_to_add`` 

147 as ``tags`` where ``classes_to_add`` is a list of strings. 

148 These classes will be added to tags of the corresponding type. 

149 """ 

150 super().__init__() 

151 self.tags = tags 

152 self._last = None 

153 self._last_tag = '' 

154 self._last_tag_unmodified = '' 

155 self._buffer = '' 

156 self._original = '' 

157 self._result = '' 

158 

159 def feed(self, data): 

160 self._original = data 

161 try: 

162 super().feed(data) 

163 res = self._result 

164 self._result = '' 

165 self._original = '' 

166 return res 

167 except ValueError: 

168 res = self._original 

169 self._result = '' 

170 self._original = '' 

171 return res 

172 

173 def error(self, message): 

174 LOGGER.error("ERROR PARSING HTML: %s", message) 

175 raise ValueError(message) 

176 

177 @staticmethod 

178 def attr_strings(attrs): 

179 return ''.join([' '+k if v is None else f" {k}={json.dumps(v)}" 

180 for k, v in attrs]) 

181 

182 def handle_starttag(self, tag, attrs): 

183 self._result += self._last_tag_unmodified + self._buffer 

184 self._buffer = '' 

185 self._last_tag_unmodified = f'<{tag}{self.attr_strings(attrs)}>' 

186 if tag in self.tags: 

187 for i, attr in enumerate(attrs): 

188 if attr[0] == 'class': 

189 classes = set(attr[1].split()).union(self.tags[tag]) 

190 attrs[i] = ' '.join(classes) 

191 break 

192 else: 

193 attrs.append(('class', ' '.join(self.tags[tag]))) 

194 self._last_tag = f'<{tag}{self.attr_strings(attrs)}>' 

195 

196 def handle_endtag(self, tag): 

197 self._result += self._last_tag + self._buffer + f'</{tag}>' 

198 self._last_tag = '' 

199 self._last_tag_unmodified = '' 

200 self._buffer = '' 

201 

202 def handle_data(self, data): 

203 self._buffer += html.escape(data) 

204 

205 def handle_startendtag(self, tag, attrs): 

206 self._result += self._last_tag_unmodified + self._buffer 

207 self._last_tag = '' 

208 self._last_tag_unmodified = '' 

209 self._buffer = '' 

210 self._result += f'<{tag}{self.attr_strings(attrs)} />' 

211 

212 

213def print_fits_headers(path): 

214 """ 

215 Read FITS file located at ``path`` and print its fits header info. 

216 """ 

217 buf = StringIO() 

218 with fits.open(path) as hdulist: 

219 buf.write("FITS Header Summary\n") 

220 buf.write("===================\n\n") 

221 hdulist.info(output=buf) 

222 buf.seek(0) 

223 result = buf.read() 

224 for i, header in enumerate(hdulist): 

225 title = f"FITS Header #{i}" 

226 result += '\n\n ' + title + '\n ' + '-'*len(title) + '\n\n' 

227 for key, value in header.header.items(): 

228 result += f' {key:<10} {value}\n' 

229 return result 

230 

231 

232def flagpreset_js(preset): 

233 """Return javascript that can be used as the ``onclick`` script for flag 

234 preset buttons. Takes radio buttons from the parent form of ``this`` whose 

235 ``name`` matches each flag name and sets them to ``checked`` if their 

236 ``value`` matches the preset's value for that flag (unchecking them if they 

237 no longer match).""" 

238 flagnames = list(preset) 

239 values = list(preset.values()) 

240 desc = json.dumps('Flag Preset description: ' + 

241 getattr(preset, 'description', "Current flag settings")) 

242 return """ 

243 var parent = this.parentElement; 

244 var desc = parent.getElementsByClassName('flag-preset-description')[0]; 

245 desc.textContent = """ + desc + """; 

246 var inputs = parent.getElementsByTagName('input'); 

247 var flagnames = """ + str(flagnames) + """; 

248 var values = """ + str(values) + """; 

249 for (var i=0; i<inputs.length; i++) { 

250 if (inputs[i].type == 'radio') { 

251 input = inputs[i]; 

252 for (var j=0; j<flagnames.length; j++) { 

253 if (input.name == flagnames[j]) { 

254 if (input.value == values[j]) { 

255 if (!input.hasAttribute('checked')) { 

256 input.setAttribute('checked', ''); 

257 input.click(); 

258 } 

259 } else { 

260 if (input.hasAttribute('checked')) { 

261 input.removeAttribute('checked'); 

262 } 

263 } 

264 } 

265 } 

266 } 

267 } 

268 """ 

269 

270 

271def jup_server(): 

272 """Get the parsed URL of a jupyter server. Return ``None`` if it isn't 

273 running or the jupyter server file is not found. 

274 """ 

275 servers = running_servers() 

276 if servers is None: 

277 return None 

278 return urlsplit(servers[0][0]) 

279 

280 

281def get_draftfile(filehandler): 

282 """Get a path to a draft file to be edited on Jupyter. 

283 """ 

284 return filehandler.eventdir+'/.'+filehandler.FILENAME+'.DRAFT' 

285 

286 

287def relative_home(run_name=None): 

288 """Get the relative path to a run based on the name of the run. Defaults 

289 to the default run directory. 

290 """ 

291 if run_name is None: 

292 return f"{LLAMA_REL_URL_HOME}/{DEFAULT_RUN_DIR_NAME}" 

293 return f"{LLAMA_REL_URL_HOME}/{PAST_RUNS_DIR_NAME}/{run_name}" 

294 

295 

296def looks_like_plaintext(path): 

297 """Check whether a path seems to be a plaintext file by seeing if the first 

298 5000 characters can be read as text. Return ``False`` if a 

299 ``FileNotFoundError`` or a ``UnicodeDecodeError`` are raised. 

300 """ 

301 try: 

302 with open(path, 'r') as infile: 

303 infile.read(5000) 

304 return True 

305 except (FileNotFoundError, UnicodeDecodeError): 

306 return False 

307 

308 

309def check_auth(username, password): 

310 """This function is called to check if a username / password combination is 

311 valid.""" 

312 return username == LLAMA_GUI_USERNAME and password == LLAMA_GUI_PASSWORD 

313 

314 

315def authenticate(): 

316 """Sends a 401 response that enables basic auth""" 

317 return Response('Could not verify your access level for that URL.\n' 

318 'You have to login with proper credentials', 401, 

319 {'WWW-Authenticate': 'Basic realm="Login Required"'}) 

320 

321 

322def queue_update(queue): 

323 """Read ``FileGraph`` instances from a ``Queue`` in a separate process and 

324 ``update`` them to avoid blocking on requests. Will call ``update`` on the 

325 messages received from ``queue``.""" 

326 while True: 

327 msg = queue.get() 

328 print("Got new message on update queue: ", msg) 

329 msg.update() 

330 print("Finished updating message: ", msg) 

331 

332 

333def queue(): 

334 """Create a new ``Queue`` for the module if it does not exist.""" 

335 if not _QUEUE_HOLDER: 

336 _QUEUE_HOLDER.append(Queue()) 

337 return _QUEUE_HOLDER[0] 

338 

339 

340def requires_auth(func): 

341 """Require basic HTTP auth for the Flask development server. This is a 

342 kludge until better WSGI routing can be implemented efficiently.""" 

343 

344 @wraps(func) 

345 def decorated(*args, **kwargs): 

346 auth = request.authorization 

347 if USE_AUTH[0] and (not auth or not check_auth(auth.username, 

348 auth.password)): 

349 return authenticate() 

350 return func(*args, **kwargs) 

351 

352 return decorated 

353 

354 

355def get_socket_fds(): 

356 """Get socket file descriptors for the current process.""" 

357 proc = psutil.Process(os.getpid()) 

358 return [c.fd for c in proc.connections()] 

359 

360 

361app = Flask(__name__) 

362LOGGER.debug("App instantiated") 

363 

364 

365def run_route(route, **kwargs): 

366 """Route requests relative to current and past runs from 

367 ``LLAMA_REL_HOME/<relative_run_dir>``. 

368 

369 Parameters 

370 ---------- 

371 route : str 

372 A ``Flask.route`` route relative to the relevant run directory, e.g. 

373 ``/<eventid>/<filename>``. 

374 **kwargs 

375 Keyword arguments for ``Flask.route``. 

376 """ 

377 LOGGER.debug("Decorator created for %s, %s", route, kwargs) 

378 

379 def decorator(func): 

380 """Register ``func`` along the specified relative routes. 

381 """ 

382 LOGGER.debug("Decorating %s", func) 

383 for reldir in RELATIVE_RUN_DIRS: 

384 full_route = LLAMA_REL_URL_HOME+reldir+route 

385 LOGGER.debug("Routing %s to %s", full_route, func) 

386 app.route(full_route, **kwargs)(func) 

387 return func 

388 

389 return decorator 

390 

391 

392def get_event_from_run(eventid, rundir=None): 

393 """Make sure that the ``eventid`` glob expression exists and is unique in 

394 this run. It is okay if it is a glob expression just so long as it uniquely 

395 specifies an event. Returns the corresponding event object. Aborts with a 

396 404 error if an event is not uniquely specified.""" 

397 kwargs = {} if rundir is None else {'rundir': rundir} 

398 events = Run(**kwargs).downselect(eventid_filter=eventid).events 

399 if not len(events) == 1: 

400 abort(404) 

401 return events[0] 

402 

403 

404def event_files_graph(event, urls): 

405 """Return a visualization of the dependency graph for this event and the 

406 current state of the files therein for embedding. ``urls`` is the URL 

407 format string to be used with the output SVG images for interactivity.""" 

408 fhm = event.files 

409 return get_graphviz(fhm, event, urls) 

410 

411 

412def cachedfilename(event, hashable, ext=''): 

413 """Get a cached filename (as an absolute path) for an event based on the 

414 current library version combined with the hash of some hashable data (which 

415 should be the object that deterministically leads to the creation of the 

416 cached output). Optionally specify a file extension to use. The data will 

417 be stored in the directory specified by the ``WEBCACHE`` variable within 

418 ``event.eventdir`` (the directory will be created if it does not exist when 

419 ``cachedfilename`` is called). This leads to automatic cache invalidation 

420 for updated code versions and distinguishes between different input 

421 arguments.""" 

422 shasum = sha256(format(hash(hashable)).encode()).hexdigest() 

423 outdir = os.path.join(event.eventdir, WEBCACHE) 

424 if not os.path.isdir(outdir): 

425 os.mkdir(outdir) 

426 return os.path.join(outdir, "v-{}-{}{}".format(__version__, shasum, ext)) 

427 

428 

429def get_graphviz(fhm, event, urls): 

430 """Fetch a graphviz svg graph, regenerating it if any of the files involved 

431 in its creation have changed in the git history.""" 

432 regen = False 

433 outfilepath = Path(cachedfilename(event, fhm, ext='.svg')) 

434 if request.args.get('del-cache', 'false') == 'true': 

435 regen = True 

436 else: 

437 paths = [Path(event.eventdir, f) 

438 for fh in fhm.values() 

439 for f in [fh.FILENAME] + list(fh.auxiliary_paths)] 

440 paths.append(outfilepath) 

441 paths.append(Path(event.eventdir, 'FLAGS.json')) 

442 t_sorted_files = sorted((p for p in paths if p.exists()), 

443 key=lambda p: p.stat().st_mtime) 

444 if not t_sorted_files: 

445 regen = True 

446 elif outfilepath != t_sorted_files[-1]: 

447 regen = True 

448 else: 

449 relevant_times = hashes(event, *paths, pretty='%ct') or [0] 

450 if outfilepath.stat().st_mtime < int(relevant_times[0]): 

451 regen = True 

452 if regen: 

453 fhm.dependency_graph(str(outfilepath), urls=urls) 

454 with open(outfilepath) as outf: 

455 return outf.read() 

456 

457 

458def request_hash(): 

459 """Get a hashable representation of the current request. Will only work for 

460 "GET" requests, otherwise a ``ValueError`` is raised. Will ignore 

461 ``request.files`` since this should only be used for uploads.""" 

462 if request.method != 'get': 

463 raise ValueError("Can only request hashable get requests.") 

464 return ( 

465 ("authorization", request.authorization), 

466 ("args", request.args), 

467 ("form", request.form), 

468 ("base_url", request.base_url), 

469 ) 

470 

471 

472def serve_file(event, filehandler, run_name=None): 

473 """Serve a summary page for this ``filehandler`` class for a specific 

474 ``eventid`` and commit (as parsed from the request arguments). If 

475 ``download=true`` appears in the query string, download this file instead 

476 of returning a summary page.""" 

477 fh = filehandler(event) 

478 if request.args.get("getsvg", 'false') == 'true': 

479 return render_template( 

480 'graphviz-plot.html', 

481 event=event, 

482 relhome=relative_home(run_name=run_name), 

483 commit=request.args.get('hash'), 

484 svg_render=lambda event, urls: filehandler_graph(event, 

485 filehandler, 

486 urls), 

487 ) 

488 if request.args.get("getfile") == 'true': 

489 return send_file( 

490 fh.fullpath, 

491 mimetype=guess_type(fh.FILENAME)[0], 

492 attachment_filename=fh.filename_for_download, 

493 as_attachment=False, 

494 ) 

495 if request.args.get("download") == 'true': 

496 return send_file( 

497 fh.fullpath, 

498 mimetype=guess_type(fh.FILENAME)[0], 

499 attachment_filename=fh.filename_for_download, 

500 as_attachment=True, 

501 ) 

502 return render_template( 

503 'file.html', 

504 pygments_styles=PYGMENTS_STYLES, 

505 run_name=run_name or DEFAULT_RUN_DIR_NAME, 

506 version=__version__, 

507 domain=LLAMA_DOMAIN, 

508 jup_url=jup_server(), 

509 processes=get_processes(), 

510 hashes=hashes, 

511 psutil=psutil, 

512 fh=fh, 

513 os=os, 

514 get_draftfile=get_draftfile, 

515 nodelegend=NODELEGEND_COLORS, 

516 relhome=relative_home(run_name=run_name), 

517 home=LLAMA_REL_URL_HOME, 

518 event=event, 

519 svg_filename=type(fh).__name__+'.svg', 

520 git_path=fh.FILENAME, 

521 hostname=gethostname(), 

522 svg_render=lambda event, urls: filehandler_graph(event, filehandler, 

523 urls), 

524 commit=request.args.get('hash'), 

525 looks_like_plaintext=looks_like_plaintext, 

526 plaintext_extensions=PLAINTEXT_EXTENSIONS, 

527 print_fits_headers=print_fits_headers, 

528 pygments=pygments, 

529 AddClassesToTerminalTags=AddClassesToTerminalTags, 

530 ) 

531 

532 

533def filehandler_graph(event, filehandler, urls): 

534 """Generate and serve the dependency graph for this specific 

535 filehandler and commit hash. The ``filehandler`` graph name is just the 

536 name of the ``filehandler`` with '.svg' appended. ``urls`` is the URL 

537 format string to be used with the output SVG images for interactivity.""" 

538 fh = filehandler(event) 

539 return get_graphviz(fh.subgraph, event, urls) 

540 

541 

542def hashes(event, *paths, **kwargs): 

543 """Safely check for git hashes on this path with ``event.git.hashes``. If a 

544 ``ValueError`` is raised, return an empty list. Used for templating where 

545 try/except is not feasible.""" 

546 try: 

547 return event.git.hashes(*paths, **kwargs) 

548 except (GitRepoUninitialized, ValueError): 

549 return [] 

550 

551 

552@run_route('/<eventid>/<filename>', methods=['GET']) 

553@requires_auth 

554def file(eventid, filename, run_name=None): 

555 """Serve a ``filehandler`` under 

556 ``relative_home(run_name)+"/<eventid>/<filename>"`` for the filename 

557 defined by this ``FileHandler`` instance. Returns a ``dict`` of registered 

558 functions. ``filename`` will be reduced to a basename.""" 

559 event = get_event_from_run_and_commit(eventid, 

560 commit=request.args.get('hash'), 

561 run_name=run_name) 

562 match = [type(f) for f in event.files.values() if f.FILENAME == filename] 

563 if match: 

564 return serve_file(event, match[0], run_name=run_name) 

565 fullpath = os.path.join(event.eventdir, os.path.basename(filename)) 

566 # serve other files as downloads 

567 if os.path.isfile(fullpath): 

568 return send_file( 

569 fullpath, 

570 mimetype=guess_type(filename)[0], 

571 attachment_filename=event.git.filename_for_download(filename), 

572 as_attachment=True, 

573 ) 

574 abort(404) 

575 

576 

577def get_processes(): 

578 """Get processes that should be running on the server for the sake of 

579 monitoring server status.""" 

580 processes = {k: list() for k in PROC_FILTERS} 

581 for pid in psutil.pids(): 

582 try: 

583 proc = psutil.Process(pid) 

584 for procname, test in PROC_FILTERS.items(): 

585 cmd = proc.cmdline() 

586 if cmd and test(cmd): 

587 processes[procname].append(proc) 

588 except (psutil.AccessDenied, psutil.ZombieProcess, 

589 psutil.NoSuchProcess): 

590 pass 

591 return {k: Proc('/processes/'+k, processes[k]) for k in processes} 

592 

593 

594@app.route('/processes/<procname>', methods=['POST']) 

595@requires_auth 

596def start_proc(procname): 

597 """Try to start or kill a process by name. If the process is not one of the 

598 processes named in ``PROC_STARTS``, abort with error 400 for a bad 

599 request.""" 

600 if request.form.get("start", "false") == "true": 

601 if procname not in PROC_STARTS: 

602 abort(400) 

603 fds = get_socket_fds() 

604 try: 

605 pid = os.fork() 

606 if pid == 0: 

607 # child process; close file descriptors 

608 for fd in fds: 

609 os.close(fd) 

610 os.system(PROC_STARTS[procname]) 

611 exit(0) 

612 else: 

613 # parent process, just return the pid and go back to what we 

614 # were doing. 

615 sleep(0.3) 

616 message = quote_plus('Ran '+PROC_STARTS[procname]) 

617 return redirect(f'/{DEFAULT_RUN_DIR_NAME}/?message={message}') 

618 except OSError as err: 

619 LOGGER.error("fork #1 failed: %d (%s)", err.errno, err.strerror) 

620 raise 

621 elif request.form.get("kill", "false") == "true": 

622 if procname not in PROC_FILTERS: 

623 abort(400) 

624 kill_list = list() 

625 for pid in psutil.pids(): 

626 proc = psutil.Process(pid) 

627 try: 

628 cmd = proc.cmdline() 

629 if cmd and PROC_FILTERS[procname](cmd): 

630 proc.kill() 

631 kill_list.append(pid) 

632 except (psutil.AccessDenied, psutil.ZombieProcess, 

633 psutil.NoSuchProcess): 

634 pass 

635 message = quote_plus('Killed '+', '.join(str(p) for p in kill_list)) 

636 return redirect(f'/{DEFAULT_RUN_DIR_NAME}/?message={message}') 

637 abort(400) 

638 

639 

640@run_route('/<eventid>/', methods=['POST']) 

641@requires_auth 

642def update_event(eventid, run_name=None): 

643 """Get ``Event.files.update`` running. If ``run_name`` is not None, treat 

644 this like a past run.""" 

645 kwargs = dict() 

646 if run_name is not None: 

647 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name) 

648 event = get_event_from_run(eventid, **kwargs) 

649 if request.form.get('CONTAINS_FLAGS', 'false') == 'true': 

650 for flagname in FlagDict.DEFAULT_FLAGS: 

651 event.flags[flagname] = request.form.get( 

652 flagname, 

653 FlagDict.DEFAULT_FLAGS[flagname] 

654 ) 

655 msg = ', '.join((k+'='+v for k, v in event.flags.items())) 

656 event.git.init() 

657 event.git.commit_changes(f"From GUI: updated flags to {msg}") 

658 message = 'message='+quote_plus("Set flags: " + msg) 

659 return redirect(request.base_url+'?'+message) 

660 if request.form.get('generate', 'false') == 'true': 

661 queue().put(event.files) 

662 message = 'message='+quote_plus('Started generating files for' + 

663 event.eventid) 

664 return redirect(request.base_url+'?'+message) 

665 abort(400) 

666 

667 

668@run_route('/<eventid>/<filename>', methods=['POST']) 

669@requires_auth 

670def upload_file(eventid, filename, run_name=None): 

671 """Manually upload a file and commit it to an event's history, overwriting 

672 the previous version. Can only upload to the latest version of the file. If 

673 no file is sent but 'lock' is 'true' or 'false', either lock or unlock the 

674 corresponding file.""" 

675 kwargs = dict() 

676 if run_name is not None: 

677 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name) 

678 event = get_event_from_run(eventid, **kwargs) 

679 match = [f for f in event.files.values() if f.FILENAME == filename] 

680 if not match: 

681 abort(404) 

682 fh = match[0] 

683 if 'file' not in request.files: 

684 # try deleting the file 

685 if request.form.get('delete', 'false') == 'true': 

686 fh.delete() 

687 message = 'message='+quote_plus('Successfully deleted '+filename) 

688 return redirect(request.base_url+'?'+message) 

689 # try generating the file 

690 if request.form.get('generate', 'false') == 'true': 

691 queue().put(fh.subgraph) 

692 # wait a moment so that the intent marker can be placed and 

693 # recognized in the updated page 

694 sleep(1) 

695 message = 'message='+quote_plus('Started generating '+filename) 

696 return redirect(request.base_url+'?'+message) 

697 # try creating a new file and editing it on Jupyter 

698 if 'edit_new_text_file' in request.form: 

699 jup_url = jup_server() 

700 draftfile = get_draftfile(fh) 

701 if not os.path.isfile(draftfile): 

702 with open(draftfile, 'w') as outfile: 

703 outfile.write( 

704 "Write thie file's contents and then save it and\n" 

705 "rename it to:\n\n" 

706 f" {fh.FILENAME}\n\n" 

707 "Do NOT rename it before you've finished it; the\n" 

708 "LLAMA update daemon will see the existing file and\n" 

709 "assume that it is ready to use.\n" 

710 ) 

711 return redirect(f"{jup_url.scheme}://{jup_url.netloc}/edit/" 

712 f"{run_name or DEFAULT_RUN_DIR_NAME}/" 

713 f"{event.eventid}/{os.path.basename(draftfile)}?" 

714 f"{jup_url.query}") 

715 # try locking/unlocking the file 

716 if 'veto' in request.form: 

717 if request.form['veto'] == 'true': 

718 fh.veto(message="Manual veto from GUI.") 

719 msg = f'Permanently vetoed {fh.FILENAME}' 

720 elif request.form['veto'] == 'false': 

721 fh.veto.unveto() 

722 msg = (f'Removed permanent veto on {fh.FILENAME} (might be ' 

723 'automatically reapplied)') 

724 else: 

725 abort(400) 

726 fh.git.init() 

727 fh.git.commit_changes(f"From GUI: {msg}") 

728 return redirect(request.base_url+'?message='+quote_plus(msg)) 

729 if 'remove_intent' in request.form: 

730 if request.form['remove_intent'] != 'true': 

731 abort(400) 

732 fh.intent.delete() 

733 change = f'Deleted intent file for {fh.FILENAME}' 

734 msg = (f"{change}; in-progress file generation will not " 

735 "be committed. You are now free to regenerate this file.") 

736 fh.git.init() 

737 try: 

738 fh.git.commit_changes(f"From GUI: {change}") 

739 except GenerationError: 

740 pass 

741 return redirect(request.base_url+'?message='+quote_plus(msg)) 

742 if 'remove_cooldown' in request.form: 

743 if request.form['remove_cooldown'] != 'true': 

744 abort(400) 

745 fh.cooldown.delete() 

746 msg = "Deleted cooldown file (consider deleting intent file too)" 

747 fh.git.init() 

748 try: 

749 fh.git.commit_changes(f"From GUI: {msg}") 

750 except GenerationError: 

751 pass 

752 return redirect(request.base_url+'?message='+quote_plus(msg)) 

753 if 'lock' in request.form: 

754 if request.form['lock'] == 'true': 

755 fh.lock.lock() 

756 change = 'Locked' 

757 elif request.form['lock'] == 'false': 

758 fh.lock.unlock() 

759 change = 'Unlocked' 

760 else: 

761 abort(400) 

762 fh.git.init() 

763 fh.git.commit_changes( 

764 "Manually {} {} from browser.".format( 

765 change.lower(), 

766 fh.FILENAME, 

767 ) 

768 ) 

769 message = 'message='+quote_plus(change+' file.') 

770 return redirect(request.base_url+'?'+message) 

771 upload = request.files['file'] 

772 if upload.FILENAME == '': 

773 # abort(418) # tfw when u r a t pot 

774 message = 'message='+quote_plus('No file selected, upload failed.') 

775 return redirect(request.base_url + '?' + 

776 request.query_string.decode() + message) 

777 if upload: 

778 upload.save(fh.fullpath) 

779 fh.git.init() 

780 fh.git.commit_changes(("Manually uploaded {} from " 

781 "browser.").format(upload.FILENAME)) 

782 message = 'message='+quote_plus('Upload successful.') 

783 return redirect(request.base_url+'?'+message) 

784 

785 

786@run_route('/', methods=['POST']) 

787@requires_auth 

788def new_event(run_name=None): 

789 """Create a new event.""" 

790 args = list() 

791 if run_name is not None: 

792 args += ['--outdir', os.path.join(PAST_RUNS_DIR, run_name, 

793 request.form['graceid'])] 

794 if 'graceid' not in request.form: 

795 abort(400) 

796 args += ['--graceid', request.form['graceid']] 

797 if request.form.get('skymap'): 

798 args += ['--skymap', request.form['skymap']] 

799 if request.form.get('overwrite') == 'true': 

800 args.append('--clobber') 

801 args.append('--flags') 

802 args += [f+'='+request.form.get(f, FlagDict.DEFAULT_FLAGS[f]) 

803 for f in FlagDict.DEFAULT_FLAGS] 

804 try: 

805 run_on_args(args, error_on_failure=True) 

806 url = (relative_home(run_name=run_name) + '/' + 

807 request.form['graceid']) 

808 except RuntimeError as err: 

809 msg = quote_plus(f'Error getting skymap with args {args}: {err}') 

810 url = f'?message={msg}' 

811 return redirect(url) 

812 

813 

814def event_version_checkout_rundir(eventid, commit, run_name=None): 

815 """Return a temporary (i.e. deletable) run directory where an 

816 ``Event.clone`` of a specific ``eventid`` ``commit`` can be cached for 

817 serving/manipulating in a consistent way while also allowing for cache 

818 flushing. Will not create any intermediate or final directories.""" 

819 kwargs = dict() 

820 if run_name is not None: 

821 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name) 

822 event = get_event_from_run(eventid, **kwargs) 

823 return os.path.join(event.eventdir, WEBCACHE, 'VERSION-CHECKOUT', commit) 

824 

825 

826def get_event_from_run_and_commit(eventid, commit=None, run_name=None): 

827 """Same as ``get_event_from_run``, but if a ``commit`` is specified, will 

828 clone the event at the specified ``commit`` in a repeatable (read: 

829 cacheable) way so that that snapshot can be used in the future (but also 

830 easily flushed).""" 

831 kwargs = dict() 

832 if run_name is not None: 

833 kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name) 

834 event = get_event_from_run(eventid, **kwargs) 

835 if commit is not None: 

836 temp_rundir = event_version_checkout_rundir(eventid, commit=commit, 

837 run_name=run_name) 

838 if not os.path.isdir(temp_rundir): 

839 os.makedirs(temp_rundir) 

840 event = event.clone(commit=commit, rundir=temp_rundir, clobber=True) 

841 # async_rm(event.eventdir, delay=7) 

842 return event 

843 

844 

845@run_route('/', methods=['GET']) 

846@requires_auth 

847def index(run_name=None): 

848 """Return a homepage for the current run.""" 

849 args = { 

850 "reverse": request.args.get('reverse', 'True') == 'True', 

851 "eventid_filter": request.args.get("eventid_filter", "S*"), 

852 } 

853 run_kwargs = dict() 

854 if run_name is not None: 

855 run_kwargs['rundir'] = os.path.join(PAST_RUNS_DIR, run_name) 

856 events = Run(**run_kwargs).downselect(sortkey=SORTKEYS['lvk'], 

857 **args).events 

858 page = int(request.args.get('page', 0)) 

859 start = EVENTS_PER_PAGE*page 

860 end = EVENTS_PER_PAGE*(page+1) 

861 downselected_events = events[start:end] 

862 pages = ceil(len(events)/float(EVENTS_PER_PAGE)) 

863 return render_template( 

864 'index.html', 

865 pygments_styles=PYGMENTS_STYLES, 

866 run_name=run_name or DEFAULT_RUN_DIR_NAME, 

867 version=__version__, 

868 domain=LLAMA_DOMAIN, 

869 jup_url=jup_server(), 

870 processes=get_processes(), 

871 hashes=hashes, 

872 psutil=psutil, 

873 nodelegend=NODELEGEND_COLORS, 

874 relhome=relative_home(run_name=run_name), 

875 home=LLAMA_REL_URL_HOME, 

876 svg_render=event_files_graph, 

877 downselected_events=downselected_events, 

878 start=start+1, 

879 end=min(end, len(events)), 

880 hostname=gethostname(), 

881 git_path='--', # include all paths in run histories 

882 total_events=len(events), 

883 pages=pages, 

884 page=page, 

885 next_page='?'+urlencode(args)+'&page='+str(page+1) if page+1 < pages else "", 

886 prev_page='?'+urlencode(args)+'&page='+str(page-1) if page > 0 else "", 

887 FlagDict=FlagDict, 

888 getattr=getattr, 

889 flagpreset_js=flagpreset_js, 

890 starting_flags=FlagDict.DEFAULT_FLAGS, 

891 starting_flag_desc=FlagDict.DEFAULT_FLAGS.description, 

892 ) 

893 

894 

895async def async_rm(path, delay=5): 

896 """Asynchronously wait for ``delay`` seconds before deleting ``path`` 

897 recursively using ``os.unlink`` for files and ``shutil.rmtree`` for 

898 directories. Ignores any ``IOError`` on the assumption that these arise 

899 from harmless tempfile deletion race conditions. Use this to clean up 

900 tempfiles.""" 

901 await asyncio.sleep(delay) 

902 try: 

903 if os.path.isdir(path): 

904 rmtree(path) 

905 else: 

906 os.unlink(path) 

907 except IOError as err: 

908 LOGGER.warning("IOError during scheduled removal of %s: %s", path, err) 

909 

910 

911@run_route('/<eventid>/') 

912@requires_auth 

913def event_page(eventid, run_name=None): 

914 """Return the event page for a given event.""" 

915 event = get_event_from_run_and_commit(eventid, 

916 commit=request.args.get('hash'), 

917 run_name=run_name) 

918 if request.args.get("getsvg", 'false') == 'true': 

919 return render_template( 

920 'graphviz-plot.html', 

921 event=event, 

922 relhome=relative_home(run_name=run_name), 

923 commit=request.args.get('hash'), 

924 svg_render=event_files_graph, 

925 ) 

926 if request.args.get("download", 'false') == 'true': 

927 with TemporaryDirectory() as tmpdir: 

928 tgz = os.path.join(tmpdir, event.eventid+'.tar.gz') 

929 event.save_tarball(tgz) 

930 return send_file( 

931 tgz, 

932 mimetype=guess_type(tgz)[0], 

933 attachment_filename=event.eventid+'.tar.gz', 

934 as_attachment=True, 

935 ) 

936 return render_template( 

937 'event.html', 

938 pygments_styles=PYGMENTS_STYLES, 

939 run_name=run_name or DEFAULT_RUN_DIR_NAME, 

940 file_previews=[getattr(event.files, f) for f in FILE_PREVIEWS], 

941 full_width_file_previews=[getattr(event.files, f) 

942 for f in FULL_WIDTH_FILE_PREVIEWS], 

943 version=__version__, 

944 domain=LLAMA_DOMAIN, 

945 jup_url=jup_server(), 

946 processes=get_processes(), 

947 hashes=hashes, 

948 psutil=psutil, 

949 event=event, 

950 nodelegend=NODELEGEND_COLORS, 

951 relhome=relative_home(run_name=run_name), 

952 home=LLAMA_REL_URL_HOME, 

953 svg_render=event_files_graph, 

954 git_path='--', # include all paths in run histories 

955 commit=request.args.get('hash'), 

956 hostname=gethostname(), 

957 plaintext_extensions=PLAINTEXT_EXTENSIONS, 

958 pygments=pygments, 

959 print_fits_headers=print_fits_headers, 

960 getattr=getattr, 

961 flagpreset_js=flagpreset_js, 

962 FlagDict=FlagDict, 

963 starting_flags={f: event.flags[f] for f in FlagDict.ALLOWED_VALUES}, 

964 starting_flag_desc="Current flag settings", 

965 AddClassesToTerminalTags=AddClassesToTerminalTags, 

966 ) 

967 

968 

969@app.route('/') 

970@requires_auth 

971def home(): 

972 """Get a home page showing available LLAMA analysis runs.""" 

973 past = {os.path.basename(k): v for k, v in past_runs().items()} 

974 return render_template( 

975 'home.html', 

976 pygments_styles=PYGMENTS_STYLES, 

977 run_name=DEFAULT_RUN_DIR_NAME, 

978 version=__version__, 

979 relhome=relative_home(None), 

980 home=LLAMA_REL_URL_HOME, 

981 domain=LLAMA_DOMAIN, 

982 jup_url=jup_server(), 

983 past_runs=past, 

984 past_runs_name=PAST_RUNS_DIR_NAME, 

985 processes=get_processes(), 

986 psutil=psutil, 

987 nodelegend=NODELEGEND_COLORS, 

988 ) 

989 

990 

991@app.route(LLAMA_REL_URL_HOME+'/static/style.css') 

992@requires_auth 

993def css(): 

994 """Get the CSS file for this app.""" 

995 return send_file( 

996 os.path.join( 

997 os.path.dirname(__file__), 

998 'static', 

999 'style.css', 

1000 ) 

1001 ) 

1002 

1003 

1004# these files do not require authorization. they are just favicons. 

1005@app.route('/static/<filename>') 

1006def rootdir_file(filename): 

1007 """Get files in '/static/root' to serve files that should live at root 

1008 (like favicons).""" 

1009 filename = os.path.basename(os.path.abspath(filename)) 

1010 dirpath = os.path.join( 

1011 os.path.dirname(__file__), 

1012 'static', 

1013 'root', 

1014 ) 

1015 if not filename in os.listdir(dirpath): 

1016 abort(404) 

1017 return send_file(os.path.join(dirpath, filename)) 

1018 

1019 

1020def get_parser(): 

1021 """Get an argument parser.""" 

1022 parser = CliParser(description=__doc__, 

1023 parents=(get_logging_cli(GUI_LOG, 'info'),)) 

1024 parser.add_argument("--use-auth", action="store_true", help=""" 

1025 Force the user to log in using basic HTTP auth. Requires that 

1026 environmental variables ``LLAMA_GUI_USERNAME`` and 

1027 ``LLAMA_GUI_PASSWORD`` are both set.""") 

1028 return parser 

1029 

1030 

1031def main(): 

1032 """Run the server.""" 

1033 parser = get_parser() 

1034 args = parser.parse_args() 

1035 USE_AUTH[0] = args.use_auth 

1036 if args.use_auth: 

1037 if LLAMA_GUI_USERNAME is None or LLAMA_GUI_PASSWORD is None: 

1038 parser.error("If `--use-auth` is provided, must set username and " 

1039 "password using environmental variables " 

1040 "`LLAMA_GUI_USERNAME` and `LLAMA_GUI_PASSWORD`") 

1041 # app.config['PROFILE'] = True 

1042 # app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=os.getcwd(), 

1043 # filename_format="{method}.{path}.{time:f}.{elapsed:06f}ms.prof", 

1044 # restrictions=[30]) 

1045 # start background process to handle file graph updates 

1046 updater_proc = Process(target=queue_update, args=(queue(),)) 

1047 updater_proc.daemon = True 

1048 updater_proc.start() 

1049 print("Started updater queue process with pid ", updater_proc.pid) 

1050 # if args.debug: 

1051 # app.run(debug=args.debug, port=args.port) 

1052 # else: 

1053 # from waitress import serve 

1054 # serve(app, listen=f'*:{args.port}') 

1055 from waitress import serve 

1056 serve(app, listen=f'*:{LLAMA_GUI_PORT}') 

1057 updater_proc.join() 

1058 

1059 

1060if __name__ == '__main__': 

1061 main()