0001 """Conversion pipeline templates. 0002 0003 The problem: 0004 ------------ 0005 0006 Suppose you have some data that you want to convert to another format, 0007 such as from GIF image format to PPM image format. Maybe the 0008 conversion involves several steps (e.g. piping it through compress or 0009 uuencode). Some of the conversion steps may require that their input 0010 is a disk file, others may be able to read standard input; similar for 0011 their output. The input to the entire conversion may also be read 0012 from a disk file or from an open file, and similar for its output. 0013 0014 The module lets you construct a pipeline template by sticking one or 0015 more conversion steps together. It will take care of creating and 0016 removing temporary files if they are necessary to hold intermediate 0017 data. You can then use the template to do conversions from many 0018 different sources to many different destinations. The temporary 0019 file names used are different each time the template is used. 0020 0021 The templates are objects so you can create templates for many 0022 different conversion steps and store them in a dictionary, for 0023 instance. 0024 0025 0026 Directions: 0027 ----------- 0028 0029 To create a template: 0030 t = Template() 0031 0032 To add a conversion step to a template: 0033 t.append(command, kind) 0034 where kind is a string of two characters: the first is '-' if the 0035 command reads its standard input or 'f' if it requires a file; the 0036 second likewise for the output. The command must be valid /bin/sh 0037 syntax. If input or output files are required, they are passed as 0038 $IN and $OUT; otherwise, it must be possible to use the command in 0039 a pipeline. 0040 0041 To add a conversion step at the beginning: 0042 t.prepend(command, kind) 0043 0044 To convert a file to another file using a template: 0045 sts = t.copy(infile, outfile) 0046 If infile or outfile are the empty string, standard input is read or 0047 standard output is written, respectively. The return value is the 0048 exit status of the conversion pipeline. 0049 0050 To open a file for reading or writing through a conversion pipeline: 0051 fp = t.open(file, mode) 0052 where mode is 'r' to read the file, or 'w' to write it -- just like 0053 for the built-in function open() or for os.popen(). 0054 0055 To create a new template object initialized to a given one: 0056 t2 = t.clone() 0057 0058 For an example, see the function test() at the end of the file. 0059 """ # ' 0060 0061 0062 import re 0063 0064 import os 0065 import tempfile 0066 import string 0067 0068 __all__ = ["Template"] 0069 0070 # Conversion step kinds 0071 0072 FILEIN_FILEOUT = 'ff' # Must read & write real files 0073 STDIN_FILEOUT = '-f' # Must write a real file 0074 FILEIN_STDOUT = 'f-' # Must read a real file 0075 STDIN_STDOUT = '--' # Normal pipeline element 0076 SOURCE = '.-' # Must be first, writes stdout 0077 SINK = '-.' # Must be last, reads stdin 0078 0079 stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ 0080 SOURCE, SINK] 0081 0082 0083 class Template: 0084 """Class representing a pipeline template.""" 0085 0086 def __init__(self): 0087 """Template() returns a fresh pipeline template.""" 0088 self.debugging = 0 0089 self.reset() 0090 0091 def __repr__(self): 0092 """t.__repr__() implements repr(t).""" 0093 return '<Template instance, steps=%r>' % (self.steps,) 0094 0095 def reset(self): 0096 """t.reset() restores a pipeline template to its initial state.""" 0097 self.steps = [] 0098 0099 def clone(self): 0100 """t.clone() returns a new pipeline template with identical 0101 initial state as the current one.""" 0102 t = Template() 0103 t.steps = self.steps[:] 0104 t.debugging = self.debugging 0105 return t 0106 0107 def debug(self, flag): 0108 """t.debug(flag) turns debugging on or off.""" 0109 self.debugging = flag 0110 0111 def append(self, cmd, kind): 0112 """t.append(cmd, kind) adds a new step at the end.""" 0113 if type(cmd) is not type(''): 0114 raise TypeError, \ 0115 'Template.append: cmd must be a string' 0116 if kind not in stepkinds: 0117 raise ValueError, \ 0118 'Template.append: bad kind %r' % (kind,) 0119 if kind == SOURCE: 0120 raise ValueError, \ 0121 'Template.append: SOURCE can only be prepended' 0122 if self.steps and self.steps[-1][1] == SINK: 0123 raise ValueError, \ 0124 'Template.append: already ends with SINK' 0125 if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): 0126 raise ValueError, \ 0127 'Template.append: missing $IN in cmd' 0128 if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): 0129 raise ValueError, \ 0130 'Template.append: missing $OUT in cmd' 0131 self.steps.append((cmd, kind)) 0132 0133 def prepend(self, cmd, kind): 0134 """t.prepend(cmd, kind) adds a new step at the front.""" 0135 if type(cmd) is not type(''): 0136 raise TypeError, \ 0137 'Template.prepend: cmd must be a string' 0138 if kind not in stepkinds: 0139 raise ValueError, \ 0140 'Template.prepend: bad kind %r' % (kind,) 0141 if kind == SINK: 0142 raise ValueError, \ 0143 'Template.prepend: SINK can only be appended' 0144 if self.steps and self.steps[0][1] == SOURCE: 0145 raise ValueError, \ 0146 'Template.prepend: already begins with SOURCE' 0147 if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): 0148 raise ValueError, \ 0149 'Template.prepend: missing $IN in cmd' 0150 if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): 0151 raise ValueError, \ 0152 'Template.prepend: missing $OUT in cmd' 0153 self.steps.insert(0, (cmd, kind)) 0154 0155 def open(self, file, rw): 0156 """t.open(file, rw) returns a pipe or file object open for 0157 reading or writing; the file is the other end of the pipeline.""" 0158 if rw == 'r': 0159 return self.open_r(file) 0160 if rw == 'w': 0161 return self.open_w(file) 0162 raise ValueError, \ 0163 'Template.open: rw must be \'r\' or \'w\', not %r' % (rw,) 0164 0165 def open_r(self, file): 0166 """t.open_r(file) and t.open_w(file) implement 0167 t.open(file, 'r') and t.open(file, 'w') respectively.""" 0168 if not self.steps: 0169 return open(file, 'r') 0170 if self.steps[-1][1] == SINK: 0171 raise ValueError, \ 0172 'Template.open_r: pipeline ends width SINK' 0173 cmd = self.makepipeline(file, '') 0174 return os.popen(cmd, 'r') 0175 0176 def open_w(self, file): 0177 if not self.steps: 0178 return open(file, 'w') 0179 if self.steps[0][1] == SOURCE: 0180 raise ValueError, \ 0181 'Template.open_w: pipeline begins with SOURCE' 0182 cmd = self.makepipeline('', file) 0183 return os.popen(cmd, 'w') 0184 0185 def copy(self, infile, outfile): 0186 return os.system(self.makepipeline(infile, outfile)) 0187 0188 def makepipeline(self, infile, outfile): 0189 cmd = makepipeline(infile, self.steps, outfile) 0190 if self.debugging: 0191 print cmd 0192 cmd = 'set -x; ' + cmd 0193 return cmd 0194 0195 0196 def makepipeline(infile, steps, outfile): 0197 # Build a list with for each command: 0198 # [input filename or '', command string, kind, output filename or ''] 0199 0200 list = [] 0201 for cmd, kind in steps: 0202 list.append(['', cmd, kind, '']) 0203 # 0204 # Make sure there is at least one step 0205 # 0206 if not list: 0207 list.append(['', 'cat', '--', '']) 0208 # 0209 # Take care of the input and output ends 0210 # 0211 [cmd, kind] = list[0][1:3] 0212 if kind[0] == 'f' and not infile: 0213 list.insert(0, ['', 'cat', '--', '']) 0214 list[0][0] = infile 0215 # 0216 [cmd, kind] = list[-1][1:3] 0217 if kind[1] == 'f' and not outfile: 0218 list.append(['', 'cat', '--', '']) 0219 list[-1][-1] = outfile 0220 # 0221 # Invent temporary files to connect stages that need files 0222 # 0223 garbage = [] 0224 for i in range(1, len(list)): 0225 lkind = list[i-1][2] 0226 rkind = list[i][2] 0227 if lkind[1] == 'f' or rkind[0] == 'f': 0228 (fd, temp) = tempfile.mkstemp() 0229 os.close(fd) 0230 garbage.append(temp) 0231 list[i-1][-1] = list[i][0] = temp 0232 # 0233 for item in list: 0234 [inf, cmd, kind, outf] = item 0235 if kind[1] == 'f': 0236 cmd = 'OUT=' + quote(outf) + '; ' + cmd 0237 if kind[0] == 'f': 0238 cmd = 'IN=' + quote(inf) + '; ' + cmd 0239 if kind[0] == '-' and inf: 0240 cmd = cmd + ' <' + quote(inf) 0241 if kind[1] == '-' and outf: 0242 cmd = cmd + ' >' + quote(outf) 0243 item[1] = cmd 0244 # 0245 cmdlist = list[0][1] 0246 for item in list[1:]: 0247 [cmd, kind] = item[1:3] 0248 if item[0] == '': 0249 if 'f' in kind: 0250 cmd = '{ ' + cmd + '; }' 0251 cmdlist = cmdlist + ' |\n' + cmd 0252 else: 0253 cmdlist = cmdlist + '\n' + cmd 0254 # 0255 if garbage: 0256 rmcmd = 'rm -f' 0257 for file in garbage: 0258 rmcmd = rmcmd + ' ' + quote(file) 0259 trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15' 0260 cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd 0261 # 0262 return cmdlist 0263 0264 0265 # Reliably quote a string as a single argument for /bin/sh 0266 0267 _safechars = string.ascii_letters + string.digits + '!@%_-+=:,./' # Safe unquoted 0268 _funnychars = '"`$\\' # Unsafe inside "double quotes" 0269 0270 def quote(file): 0271 for c in file: 0272 if c not in _safechars: 0273 break 0274 else: 0275 return file 0276 if '\'' not in file: 0277 return '\'' + file + '\'' 0278 res = '' 0279 for c in file: 0280 if c in _funnychars: 0281 c = '\\' + c 0282 res = res + c 0283 return '"' + res + '"' 0284 0285 0286 # Small test program and example 0287 0288 def test(): 0289 print 'Testing...' 0290 t = Template() 0291 t.append('togif $IN $OUT', 'ff') 0292 t.append('giftoppm', '--') 0293 t.append('ppmtogif >$OUT', '-f') 0294 t.append('fromgif $IN $OUT', 'ff') 0295 t.debug(1) 0296 FILE = '/usr/local/images/rgb/rogues/guido.rgb' 0297 t.copy(FILE, '@temp') 0298 print 'Done.' 0299
Generated by PyXR 0.9.4