1# (c) Stefan Countryman, 2019 

2 

3""" 

4Tools for dealing with metadata of FileHandlers. 

5""" 

6 

7import uuid 

8import functools 

9import time 

10from pathlib import Path 

11from datetime import datetime 

12from dateutil.parser import parse as parsetime 

13from llama.utils import ( 

14 __version__, 

15) 

16from llama.classes import RiderFile, JsonRiderMixin 

17from llama.flags import FlagDict 

18 

19 

20class MetaData(RiderFile, JsonRiderMixin): 

21 """ 

22 A class for getting metadata about the output file of a ``FileHandler`` 

23 and determining things like file obsolescence. 

24 """ 

25 

26 rider_fmt = '.{}.metadata.json' 

27 

28 def write(self, wall_time): 

29 """ 

30 Write a metadata file for this ``FileHandler``. Almost all metadata 

31 about a generation attempt can be determined from either the LLAMA 

32 execution context or the calling ``FileHandler``; the exception is the 

33 ``wall_time``, i.e. time spent generating this output file; this value 

34 must be provided as an argument. 

35 """ 

36 sums, sub_sums = list(self.manifest_filehandlers)[0].dep_checksums() 

37 metadata = { 

38 'checksums': {f.clsname: f.checksum() 

39 for f in self.manifest_filehandlers}, 

40 'dep_checksums': sums, 

41 'dep_subset_checksums': sub_sums, 

42 'version': __version__, 

43 'manifest_filehandlers': format(self.manifest_filehandlers), 

44 'wall_time': wall_time, 

45 'generation_uuid': str(uuid.uuid1()), 

46 'generation_time': datetime.utcnow().isoformat(), 

47 'flags': dict(FlagDict(self.eventdir)), 

48 } 

49 self.write_json(metadata) 

50 

51 @property 

52 def dep_subset_checksums(self): 

53 """ 

54 Read the stored ``sha256`` sums of the **subset** of the contents of 

55 the input files (i.e. ``DEPENDENCIES``) for this ``MetaData`` 

56 instance's ``manifest_filehandlers``. The keys of this ``dict`` are 

57 ``FileHandler.clsname`` values and the values are the corresponding sha 

58 sums computed from the **subet** of the input data that is actually 

59 used by the ``generate`` methods of each ``FileHandler`` in 

60 ``manifest_filehandlers``. See ``FileHandler.dep_checksums`` return 

61 values for details and use cases. 

62 

63 Raises 

64 ------ 

65 FileNotFoundError 

66 If the metadata file does not exist. 

67 KeyError 

68 If the file format of the metadata files has changed and they do 

69 not contain the hash data. 

70 """ 

71 return self.read_json(err=True)['dep_subset_checksums'] 

72 

73 @property 

74 def dep_checksums(self): 

75 """ 

76 Read the stored ``sha256`` sums of the contents of the input files 

77 (i.e. ``DEPENDENCIES``) for this ``MetaData`` instance's 

78 ``manifest_filehandlers``. The keys of this ``dict`` are 

79 ``FileHandler.clsname`` values and the values are the corresponding sha 

80 sums. See ``FileHandler.dep_checksums`` return values for details and 

81 use cases. 

82 

83 Raises 

84 ------ 

85 FileNotFoundError 

86 If the metadata file does not exist. 

87 KeyError 

88 If the file format of the metadata files has changed and they do 

89 not contain the hash data. 

90 """ 

91 return self.read_json(err=True)['dep_checksums'] 

92 

93 @property 

94 def checksums(self): 

95 """ 

96 Read the stored ``sha256`` checksums of the contents of this 

97 ``MetaData`` instance's ``manifest_filehandlers``. The keys of this 

98 ``dict`` are ``FileHandler.clsname`` values and the values are the 

99 corresponding checksums. 

100 

101 Raises 

102 ------ 

103 FileNotFoundError 

104 If the metadata file does not exist. 

105 KeyError 

106 If the file format of the metadata files has changed and they do 

107 not contain the hash data. 

108 IOError 

109 If the checksums have not been computed for all files in the 

110 manifest. 

111 """ 

112 checksums = self.read_json(err=True)['checksums'] 

113 # don't return an incomplete or overcomplete list; instead, 

114 # recompute 

115 if {f.clsname for f in self.manifest_filehandlers} == set(checksums): 

116 return checksums 

117 raise IOError("Could not load all checksums. Existing checksums: ", 

118 format(checksums)) 

119 

120 @property 

121 def generation_uuid(self): 

122 """ 

123 Get a UUID for this generation event. Raises an ``IOError`` if the 

124 metadata file does not exist and a ``KeyError`` if the 

125 ``generation_uuid`` information is not present in the metadata file. 

126 """ 

127 return self.read_json(err=True)['generation_uuid'] 

128 

129 @property 

130 def wall_time(self): 

131 """ 

132 Time in seconds spent running ``_generate`` to make this file. 

133 Raises an ``IOError`` if the metadata file does not exist and a 

134 ``KeyError`` if the ``wall_time`` information is not present in the 

135 metadata file. 

136 """ 

137 return self.read_json(err=True)['wall_time'] 

138 

139 @property 

140 def generation_time(self): 

141 """ 

142 Get the ``datetime`` object representing the UTC time at which this 

143 file was generated according to its rider metadata file. Raises an 

144 ``IOError`` if the metadata file does not exist and a ``KeyError`` if 

145 the ``generation_time`` information is not present in the metadata 

146 file. 

147 """ 

148 return parsetime(self.read_json(err=True)['generation_time']) 

149 

150 @property 

151 def flags(self): 

152 """ 

153 Return a dictionary of the flags defined for this event at the time 

154 when the file was generated. 

155 """ 

156 return self.read_json(err=True)['flags'] 

157 

158 

159class MetaDataMixin: 

160 """ 

161 Add a ``meta`` class to an object along with a checkin decorator. 

162 """ 

163 

164 @property 

165 def meta(self): 

166 """ 

167 Metadata for this file produced during automated generation attempts. 

168 """ 

169 return MetaData(self.eventdir, self.manifest_filehandlers) 

170 

171 @staticmethod 

172 def decorate_checkin(func): 

173 """ 

174 Copy metadata from the temporary generation directory to the parent 

175 directory as part of checkin. 

176 """ 

177 

178 @functools.wraps(func) 

179 def wrapper(self, gen_result, *args, **kwargs): 

180 """ 

181 Copy metadata from the temporary generation directory to the parent 

182 directory as part of checkin. 

183 """ 

184 result = func(self, gen_result, *args, **kwargs) 

185 for metaname in self.meta.filenames: 

186 src = Path(gen_result.fh.eventdir, metaname) 

187 dest = Path(self.eventdir, metaname) 

188 dest.write_bytes(src.read_bytes()) 

189 return result 

190 

191 return wrapper 

192 

193 @staticmethod 

194 def decorate_generate(func): 

195 """ 

196 Record generation wall time and other metadata for a successful 

197 generation attempt. 

198 """ 

199 

200 @functools.wraps(func) 

201 def wrapper(self, *args, **kwargs): 

202 """ 

203 Record generation wall time and other metadata for a successful 

204 generation attempt. 

205 """ 

206 start = time.time() 

207 result = func(self, *args, **kwargs) 

208 if result.err is None: 

209 self.meta.write(wall_time=time.time()-start) 

210 return result 

211 

212 return wrapper