magma.magma

Source Code for Package magma.magma

1 #!/usr/bin/env python 2 # encoding: utf-8 3 4 # MAGnet MAnifest Management - Readout and create lists of magnets in yaml format. 5 # 6 # Copyright © 2008 Arne Babenhauserheide 7 # 8 # This program is free software: you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation, either version 3 of the License, or 11 # (at your option) any later version. 12 # 13 # This program is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with this program. If not, see <http://www.gnu.org/licenses/> 20 21 22 """MAGnet MAnifest Management - Readout and create lists of magnets in yaml format. 23 24 Magma lists are lists of files which can be downloaded via magnet links. 25 26 They are written in yaml format to be easily readable as well as flexible and powerful. 27 28 29 A simple Magma file looks like the following:: 30 #MAGMAv0.4 31 files: 32 - filename: input_file.txt 33 urn: 34 sha1: 3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G 35 - filename: input_file2.txt 36 urn: 37 sha1: 3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G 38 39 40 This script parses the basic features of MAGMAv0.4, which is being specified at the moment. 41 It doesn't parse the current MAGMAv0.2 files! 42 43 Use it only, if you want to take a peek at what the updated specification will bring. 44 45 46 It depends on U{pyyaml <http://pyyaml.org/>}, tagpy and IPy. 47 48 An example magma file is I{example-0.4.magma}. 49 50 This API documentation is avaible from U{http://gnuticles.gnufu.net/pymagma/}, and the code is avaible from a U{Mercurial repository <http://freehg.org/u/ArneBab/magma/>}. 51 52 The module can also be found in the U{Python Package Index (PyPI) <http://pypi.python.org/pypi/magma/>}. It is available under the GPL-3 or later. 53 54 It is being written by U{Arne Babenhauserheide <http://draketo.de>}. If you wish to contribute, please drop me a mail with the word "tunnel" somewhere in the subject line (to get past my spamfilter). My email: arne_bab at web de . 55 56 Usage of this module 57 ==================== 58 59 >>> # from magma import * 60 >>> magma = open_file("example-0.4.magma") 61 62 or 63 64 >>> magma = create_from_input_files(["input_file.txt", "input_file2.txt"]) 65 66 67 API / usage 68 =========== 69 70 B{Creating magma lists from input files} 71 72 Create a list from input files. 73 >>> magma = create_from_input_files(["input_file.txt"]) 74 75 Output the list, showing the empty entries, which could be filled with data. 76 >>> print magma 77 #MAGMAv0.4 78 files: 79 - filename: input_file.txt 80 gnutella: 81 alt-locs: [] 82 urls: [] 83 urn: 84 sha1: 3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G 85 <BLANKLINE> 86 # Magmav0.2 compatibility section 87 list: 88 - "magnet:?xt=urn%3Asha1%3A3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G&dn=input_file.txt" 89 <BLANKLINE> 90 91 And save it to a file with the empty entries cleaned away. 92 >>> magma.save(path="example.magma") 93 94 95 Or just return a list of its files as MagmaFiles. 96 >>> files = magma.files 97 98 Or output all magnet links inside the magma list. 99 >>> print magma.magnets 100 ['magnet:?xt=urn%3Asha1%3A3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G&dn=input_file.txt'] 101 102 103 B{Reading from magma lists} 104 105 Readout the data from a file. 106 >>> magma = open_file("example.magma") 107 108 Now get the files inside the list. 109 >>> files = magma.files 110 >>> for i in files: print i 111 input_file.txt 3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G 112 113 Get the first file from the list. 114 >>> one_file = files[0] 115 >>> print one_file 116 input_file.txt 3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G 117 118 And output its magnet. 119 >>> print one_file.magnet 120 magnet:?xt=urn%3Asha1%3A3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G&dn=input_file.txt 121 122 Or output all its (meta-) data. 123 >>> print one_file.data 124 {'urn': {'sha1': '3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G'}, 'gnutella': {'alt-locs': []}, 'urls': [], 'filename': 'input_file.txt'} 125 126 127 Output all magnet links inside the magma list. 128 >>> print magma.magnets 129 ['magnet:?xt=urn%3Asha1%3A3UJCLAOIZVCNAIT7TQYFLAP7ZNFW6G2G&dn=input_file.txt'] 130 131 Load a Magma object from a string containing Magma data via magma.load(). 132 As test, load a minimal Magma file, containing nothing but the header. 133 >>> magma = load("#MAGMAv0.4") 134 135 And cry out, if the string doesn't begin with the Magma header (isn't Magma data!). 136 >>> magma = load("a:b") 137 Traceback (most recent call last): 138 ... 139 AssertionError: Doesn't begin with Magma v0.4 header #MAGMAv0.4 140 141 Also dump yaml data via magma.dump() 142 >>> print dump(magma) 143 #MAGMAv0.4 144 {} 145 <BLANKLINE> 146 # Magmav0.2 compatibility section 147 list: 148 <BLANKLINE> 149 150 Or just print the Magma data. 151 >>> print magma 152 #MAGMAv0.4 153 {} 154 <BLANKLINE> 155 # Magmav0.2 compatibility section 156 list: 157 <BLANKLINE> 158 159 160 Plans: Gnutella 161 - Also pass metadata for the magma file and include that alongside the file list. 162 - Add Alt-locs (depends on the network used). Maybe GNet-Alt-Loc for Gnutella. Gets parsed to magnet xs automatically. 163 - Alt-locs at ["sources"]["gnutella"]["alt-locs"] instead of ["gnutella"]["alt-locs"] -> general sources possible. list of HTTP sources at: ["sources"]["http"] 164 165 Ideas: 166 - Readout metadata from files, for example using the taglib, pyPdf or such. - partly done - 167 - If given both magma list and files, the magma list gets extended by the files. 168 - Joining and seperating magma lists. 169 - Passing several additional input lists to join with the main list. 170 - Passing the data of files along with magma metadata to create the list. 171 - Passing the indizes of files to automatically slice the list. maybe even with the syntax sliced_magma = magma[a:b] 172 - If a dir gets passed, add all files within that dir. Also a parameter to ask, if it should be done recursively. 173 - If hashing files takes very long (for example seen with clock.tick()), give more output. 174 - Add more commandline switches (verbose, help, ... - see the babtools_gentoo script) 175 176 """ 177 178 #### Background #### 179 180 __author__ = 'Arne Babenhauserheide' 181 __copyright__ = 'Copyright (C) 2008 Arne Babenhauserheide' 182 __date__ = '2008-05-09' 183 __licence__ = 'GPL-3 or later' 184 __program__ = 'MAGnet MAnifest Management' 185 __version__ = '0.3.9' 186 187 # The dependencies of the __init__ only. 188 __depends__ = '' 189

190 -def parse_dependencies(init_depends):

191 """Aggregate the dependencies from all submodules. 192 193 @param init_depends: The dependencies of the __init__ file. 194 @type init_depends: String 195 @return: A string with dependencies, seperated by ', '. 196 """ 197 # aggregate dependencies from all submodules. 198 dep_list = [init_depends] 199 from create_simple_magma_list import __depends__ 200 dep_list.append(__depends__) 201 from magma_list import __depends__ 202 dep_list.append(__depends__) 203 from sha1_gnutella import __depends__ 204 dep_list.append(__depends__) 205 206 # Now aggregate it, simply using a set which discards multiple entries. 207 deps = set() 208 for i in dep_list: 209 # We need to turn the individual __depends__ into lists to add them. 210 [deps.add(j) for j in i.split(", ")] 211 # And discard a possibly included emtpy string (if there is no "", nothing happens). 212 deps.discard("") 213 214 # Now assign it as string to the __depends__ 215 return ", ".join(deps)

216 217 __depends__ = parse_dependencies(__depends__) 218 219 220 #### Background #### 221 222 223 #### Functions #### 224 225 from magma_list import Magma 226

227 -def open_file(filepath):

228 """Open a file as Magma list and produce the corresponding Python object. 229 230 @param filepath: The path to a Magma file. 231 @type filepath: String 232 @return: A Magma Object. 233 """ 234 return Magma(magma_file=filepath)

235

236 -def create_from_input_files(files):

237 """Create a magma file from given input files. 238 239 @param files: A list of file paths 240 @type files: List of Strings 241 @return: A Magma Object.""" 242 return Magma(input_files=files)

243

244 -def load(yaml_data):

245 """Parse a Magma list and produce the corresponding Python object. 246 247 @param yaml_data: Data read from a yaml file 248 @type yaml_data: String 249 @return: A Magma Object. 250 """ 251 return Magma(yaml_data=yaml_data)

252

253 -def dump(magma):

254 """Return the string representation of the Magma file. 255 256 @param magma: A Magma object. 257 @type magma: magma.Magma 258 @return: A String represenation of the Magma file (in yaml format). 259 """ 260 return magma.__str__()

261 262 263 264 265 #### Self-Test #### 266

267 -def _test():

268 """Do all doctests. 269 270 @return: None 271 """ 272 # Get the testmod function from the doctest module. 273 from doctest import testmod 274 # And call it. 275 testmod()

276 277 # If this script gets called directly, do the doctests. 278 if __name__ == "__main__": 279 _test() 280