1# (c) Stefan Countryman, 2019 

2 

3import json 

4from llama.cli import get_logging_cli, CliParser 

5from llama.dev.upload import ( 

6 __doc__, 

7 upload_and_get_manifest, 

8 DEFAULT_ROOT, 

9 DEFAULT_GLOB, 

10 DEFAULT_KEY_ROOT, 

11) 

12from llama.com.s3 import DEFAULT_BUCKET 

13 

14__doc__ += """ 

15Prints out a manifest as a JSON dictionary mapping local filenames to tuples of 

16corresponding remote URLs and file hashes. 

17""" 

18 

19 

20def get_parser(): 

21 """Get CLI parser.""" 

22 parser = CliParser(description=__doc__, 

23 parents=(get_logging_cli('/dev/null', 'info'),)) 

24 parser.add_argument("-r", "--root", default=DEFAULT_ROOT, help=""" 

25 The root directory to upload to remote storage. (default: current 

26 directory). Paths in the printed manifest will be relative to this.""") 

27 parser.add_argument("-g", "--glob", default=DEFAULT_GLOB, help=""" 

28 A glob matching files in ``--root`` that should be uploaded and thrown 

29 in the manifest. (default: all files in all subdirectories).""") 

30 parser.add_argument("-p", "--prefix", default=DEFAULT_KEY_ROOT, help=f""" 

31 A prefix to the key used on the remote storage instance. This can be 

32 something pathlike with ``/`` indicating a directory, but note that it 

33 is just a prefix prepended to the front of the relative paths found in 

34 ``--root`` matching ``--glob``, so you'll have to add a trailing slash 

35 if you want this to be a containing directory. (default: 

36 {DEFAULT_KEY_ROOT})""") 

37 parser.add_argument("-b", "--bucket", default=DEFAULT_BUCKET, help=f""" 

38 The storage bucket to upload to. For DigitalOcean Spaces, this is just 

39 the name of the directory in that space's top-level directory. 

40 (default: {DEFAULT_BUCKET})""") 

41 parser.add_argument("-P", "--private", action="store_true", help=""" 

42 Upload this file privately, leaving no public link. You probably don't 

43 want to do this unless you're just dumping data onto a storage space to 

44 deal with later.""") 

45 parser.add_argument("-R", "--relpath", action="store_true", help=""" 

46 If specified, then the relative path of files from ``--root`` will 

47 become part of their key on the remote object store, sandwiched between 

48 the ``--prefix`` and the file's sha256sum. Use this if you are manually 

49 uploading something to the object store that you want to keep track of 

50 outside of source-code, e.g. one-off build artifacts and the like. You 

51 should *AVOID* using this flag if you're just interested in throwing 

52 files in an object store and organizing them with the returned manifest 

53 in a version-controlled way, since objects are by default only named 

54 after their hash. This lets you use the manifest to specify where files 

55 go on the filesystem without duplicating files on the remote object 

56 store.""") 

57 parser.add_argument("--dry-run", action='store_true', help=""" 

58 If specified, just print the manifest that would result from an upload 

59 and quit. Use this to see where your files will be uploaded before 

60 actually doing it.""") 

61 return parser 

62 

63 

64def main(): 

65 """Run CLI.""" 

66 parser = get_parser() 

67 args = parser.parse_args() 

68 manifest = upload_and_get_manifest(root=args.root, glob=args.glob, 

69 key_prefix=args.prefix, 

70 key_uses_relpath=args.relpath, 

71 bucket=args.bucket, 

72 public=(not args.private), 

73 dry_run=args.dry_run) 

74 print(json.dumps(manifest, indent=4, sort_keys=True)) 

75 

76 

77if __name__ == "__main__": 

78 main()