1# (c) Stefan Countryman, 2019 

2 

3""" 

4Scripts for setting environmental and configuration settings and fetching data 

5files that were too large to be included with the package distribution (or, 

6alternatively, were access-controlled separately from the package 

7distribution). 

8 

9This method of file storage makes it possible to install a much smaller 

10distribution of pure code and only download pipeline data files as necessary. 

11It also makes it possible to lazy-download the correct file version as-needed. 

12 

13The ``DESTINATIONS`` module-level variable maps semantic names of file download 

14destinations (as used in the command-line interface) to tuples of local 

15root directories for storing downloaded files and manifests specifying which 

16files to download and where to situate them relative to those root directories. 

17""" 

18 

19from pathlib import Path 

20from hashlib import sha256 

21import logging 

22from llama.install.manifest import MANIFEST 

23from llama.com.dl import download 

24from llama.utils import DATADIR 

25 

26LOGGER = logging.getLogger(__name__) 

27DESTINATIONS = { 

28 'data': (DATADIR, MANIFEST), 

29} 

30 

31 

32def install_manifest(root: str, manifest: dict): 

33 """Download files as specified in the ``manifest`` dict (as returned by 

34 ``llama.dev.upload.upload_and_get_manifest``) under the ``root`` 

35 directory. Existing files with the expected hash will not be 

36 redownloaded, but **existing files will be overwritten**, so exercise 

37 caution when calling this function. Files in ``root`` not mentioned in the 

38 manifest will be ignored; delete them manually if they are no longer 

39 needed. Also places the ``manifest`` in the ``root`` directory, making it 

40 possible to check whether post-install downloads are up-to-date.""" 

41 LOGGER.debug(f"Downloading to root {root} from manifest {manifest}") 

42 rootdir = Path(root) 

43 for dest, info in manifest.items(): 

44 outfile = rootdir/dest 

45 url, shasum = info 

46 if outfile.is_file(): 

47 LOGGER.debug(f"{outfile} already exists, checking if sha256 sum is" 

48 "expected: {shasum}") 

49 with open(outfile, 'rb') as outf: 

50 actual_shasum = sha256(outf.read()).hexdigest() 

51 if shasum == actual_shasum: 

52 LOGGER.debug("sha256sum matches, continuing to next file.") 

53 continue 

54 LOGGER.debug("sha256sum mismatch with existing file " 

55 f"{outfile} -- expected: {shasum} actual: " 

56 f"{actual_shasum}") 

57 LOGGER.info("Existing file {outfile} is corrupt or out of date " 

58 "(wrong sha256sum), redownloading.") 

59 download(url, str(outfile.resolve()), sha256sum=shasum, mkdirs=True) 

60 LOGGER.info("Finished downloading files from manifest.")