MAGMA  magma-1.4.0
Matrix Algebra on GPU and Multicore Architectures
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups
codegen.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 """@package Tools
3 
4 This python script is responsible for precision generation replacements
5 as well as replacements of any kind in other files. Different types of
6 replacements can be defined such that no two sets can conflict. Multiple
7 types of replacements can, however, be specified for the same file.
8 
9 @author Wesley Alvaro
10 @date 2011-4-8
11 
12 """
13 __author__="alvaro"
14 __date__ ="$Sep 2, 2010 10:09:19 AM$"
15 __version__=11.0408
16 
17 import sys;
18 import re;
19 import shlex;
20 import os;
21 from os import path;
22 from optparse import OptionParser,OptionGroup;
23 from magmasubs import subs;
24 from datetime import datetime;
25 
26 """Keyword used to signal replacement actions on a file"""
27 KEYWORD = '@precisions';
28 """Replace the above keyword with this one post-replacements"""
29 DONE_KEYWORD = '@generated';
30 """Regular expression for the replacement formatting"""
31 REGEX = '^.*'+KEYWORD+'\s+((\w+,?)+)\s+(\w+)\s+->\s*((\s\w+)+).*$';
32 """Default acceptable extensions for files during directory walking"""
33 EXTS = ['.c','.h','.f'];
34 
35 def check_gen(file, work, rex):
36  """Reads the file and determines if the file needs generation."""
37  fd = open(path.realpath(file), 'r');
38  lines = fd.readlines();
39  fd.close();
40  for line in lines:
41  m = rex.match(line);
42  if m is None: continue;
43  work.append((file, m.groups(), ''.join(lines)));
44 
45 def grep(string,list):
46  expr = re.compile(string)
47  return filter(expr.search,list)
48 
49 def hidden(file):
50  """Exclude hidden files"""
51  return not file.startswith('.');
52 
53 def valid_extension(file):
54  """Exclude non-valid extensions"""
55  global EXTS;
56  for ext in EXTS:
57  if file.endswith(ext):
58  return True;
59  return False;
60 
61 def relpath(p):
62  """Get the relative path of a file."""
63  p = path.realpath(p);
64  return p.replace(path.realpath('.')+'/','');
65 
66 class Conversion:
67  """
68  This class works on a single file to create generations
69  """
70  """Static. Is the conversion in debug mode? More verbose."""
71  debug = False;
72  """Static. Is the conversion in test mode? No real work."""
73  test = False;
74  """Static. Is the conversion in make mode? Output make commands."""
75  make = False;
76  """Static. What (if any) prefix is specified for the output folder?
77  If None, use the file's resident folder."""
78  prefix = None;
79  required_precisions = [];
80  """Static. A running list of files that are input."""
81  files_in = [];
82  """Static. A running list of files that are output."""
83  files_out = [];
84 
85  def __init__(self, file = None, match = None, content = None):
86  """Constructor that takes a file, match, and content.
87  @param file The file name of the input.
88  @param match The regular expression matches
89  @param content The ASCII content of the file.
90  """
91  if file is None: return;
92  self.content = content;
93  file = path.realpath(file);
94  rel = relpath(file);
95  self.file = list(path.split(file));
96  self.date = path.getmtime(file);
97  if sys.platform!="win32" and path.samefile(path.join(self.file[0],self.file[1]),sys.argv[0]):
98  raise ValueError('Let\'s just forget codegen.py');
99  try:
100  """['normal','all','mixed'] for example. This(ese) are the replacement types to be used."""
101  self.types = match[0].split(',');
102  """'z' for example. This is the current file's `type`."""
103  self.precision = match[2].lower();
104  """['c','d','s'] for example. This is the current file's destination `types`."""
105  self.precisions = match[3].lower().split();
106  if len(self.required_precisions):
107  self.precstmp = [];
108  for prec in self.required_precisions:
109  if prec in self.precisions:
110  self.precstmp.append(prec);
111  self.precisions = self.precstmp;
112  except:
113  raise ValueError(path.join(self.file[0],self.file[1])+' : Invalid conversion string');
114  self.files_in.append(rel);
115 
116 
117  def run(self):
118  """Does the appropriate work, if in test mode, this is limited to only converting names."""
119  if self.convert_names() and not self.test:
120  """If not in test mode, actually make changes and export to disk."""
121  self.convert_data();
122  self.export_data();
123 
124  def convert_names(self):
125  """Investigate file name and make appropriate changes."""
126  self.names = [];
127  self.dates = [];
128  self.converted = [];
129  load = False;
130  if self.debug: print '|'.join(self.types), self.precision, relpath(path.join(self.file[0],self.file[1]));
131  for precision in self.precisions:
132  """For each destination precision, make the appropriate changes to the file name/data."""
133  new_file = self.convert(self.file[1], precision);
134  if self.debug: print precision,':',
135  if new_file <> self.file[1]:
136  """A filename change is required for the conversion to continue."""
137  if self.prefix is None:
138  """If no prefix is specified, use the file's current folder."""
139  prefix = self.file[0]
140  makeprefix = '';
141  else:
142  """If a prefix is specified, set it up."""
143  prefix = self.prefix;
144  makeprefix = '--prefix '+prefix;
145  """Where the destination file will reside."""
146  conversion = path.join(prefix, new_file);
147  file_out = relpath(conversion);
148  if self.make:
149  """If in GNU Make mode, write the rule to create the file."""
150  file_in = relpath(path.join(self.file[0],self.file[1]));
151  print file_out+':',file_in;
152  print "\t$(PYTHON)",path.realpath(sys.argv[0]),makeprefix,'-p',precision,"--file",file_in;
153  self.names.append(new_file);
154  self.files_out.append(file_out);
155  if self.debug: print relpath(conversion), ':',
156  try:
157  """Try to emulate Make like time based dependencies."""
158  date = path.getmtime(conversion);
159  diff = self.date - date;
160  self.dates.append(diff);
161  if self.debug:
162  if diff > 0: print 'Old',
163  else: print 'Current',
164  print diff;
165  if diff > 0: load = True;
166  except:
167  if self.debug: print 'Missing';
168  self.dates.append(None);
169  load = True;
170  else:
171  """There was no change in the file's name, thus,
172  no work can be done without overwriting the original."""
173  if self.debug: print '<No Change>',':';
174  else: print >> sys.stderr, new_file, 'had no change for', precision;
175  self.names.append(None);
176  self.dates.append(None);
177  return load;
178 
179  def export_data(self):
180  """After all of the conversions are complete,
181  this will write the output file contents to the disk."""
182  for i in range(len(self.names)):
183  name = self.names[i];
184  data = self.converted[i];
185  if data is None or name is None: continue;
186  if self.prefix is None:
187  fd = open(path.join(self.file[0],name), 'w');
188  else:
189  fd = open(path.join(self.prefix,name), 'w');
190  fd.write(data);
191  fd.close();
192 
193 
194  def convert_data(self):
195  """Convert the data in the files by making the
196  appropriate replacements for each destination precision."""
197  for i in range(len(self.precisions)):
198  precision = self.precisions[i];
199  name = self.names[i];
200  date = self.dates[i];
201  if name is not None and (date is None or date > 0):
202  self.converted.append(self.convert(self.content, precision));
203  else: self.converted.append(None);
204 
205  def substitute(self, sub_type, data, precision):
206  """This operates on a single replacement type.
207  @param sub_type The name of the replacement set.
208  @param data The content subject for replacments.
209  @param precision The target precision for replacements.
210  """
211  try:
212  """Try to select the requested replacements."""
213  work = subs[sub_type];
214  prec_to = work[0].index(precision);
215  prec_from = work[0].index(self.precision);
216  except:
217  """If requested replacement type does not exist,
218  return unaltered contents."""
219  return data;
220  for i in range(1,len(work)):
221  """Requested replacements were found,
222  execute replacements for each entry."""
223  try:
224  search = work[i][prec_from];
225  replace = work[i][prec_to];
226  if not search: continue;
227  replace = replace.replace('\*','*');
228  if sub_type != 'tracing' :
229  replace = replace.replace('\(','(');
230  replace = replace.replace('\)',')');
231  data = re.sub(search, replace, data);
232  except:
233  print 'Bad replacement pair ',i,'in',sub_type;
234  continue;
235  return data;
236 
237  def convert(self, data, precision):
238  """Select appropriate replacements for the current file.
239  @param data The content subject for the replacements.
240  @param precision The target precision for generation.
241  """
242  global KEYWORD, DONE_KEYWORD;
243  try:
244  """All files undergo the "all" replacements."""
245  data = self.substitute('all', data, precision);
246  except: pass;
247  for sub_type in self.types:
248  """For all the other types of conversion for the current file,
249  make the correct replacements."""
250  if sub_type == 'all': continue;
251  try:
252  data = self.substitute(sub_type, data, precision);
253  except Exception, e:
254  raise ValueError('I encountered an unrecoverable error while working in subtype:',sub_type+'.');
255  """Replace the replacement keywork with one that signifies this is an output file,
256  to prevent multiple replacement issues if run again."""
257  data = re.sub(KEYWORD+' '+','.join(self.types)+'.*', DONE_KEYWORD+' '+precision+' '+datetime.now().ctime(), data);
258  return data;
259 
260 def main():
261  """Create option parser, set static variables of the converter and manage printing options/order."""
262  global REGEX, EXTS;
263  """The compiled regular expression for detecting proper files."""
264  rex = re.compile(REGEX);
265  """Files found to be workable."""
266  work = [];
267 
268  """Create the options parser for detecting options on the command line."""
269  parser = OptionParser(usage="Usage: %prog [options]",version='%prog '+str(__version__));
270  group = OptionGroup(parser,"Printing Options","These options control the printing output.");
271  group.add_option("-i", "--in-files", help='Print the filenames of files for precision generation.', action='store_true', dest='in_print', default=False);
272  group.add_option("-o", "--out-files", help='Print the filenames for the precision generated files.', action='store_true', dest='out_print', default=False);
273  group.add_option("-m", "--make", help='Spew a GNU Make friendly file to standard out.', action='store_true', dest='make', default=False);
274  group.add_option("-d", "--debug", help='Print debugging messages.', action='store_true', dest='debug', default=False);
275  parser.add_option_group(group);
276  group = OptionGroup(parser,"Operating Mode Options","These options alter the way the program operates on the input/output files.");
277  group.add_option("-c", "--clean", help='Remove the files that are the product of generation.', action='store_true', dest='out_clean', default=False);
278  group.add_option("-T", "--test", help='Don\'t actually do any work.', action='store_true', dest='test', default=False);
279  parser.add_option_group(group);
280  group = OptionGroup(parser,"Settings","These options specify how the work should be done.");
281  group.add_option("-P", "--prefix", help='The output directory if different from the input directory.', action='store', dest='prefix', default=None);
282  group.add_option("-f", "--file", help='Specify files on which to operate.', action='store', dest='fileslst', type='string', default=None );
283  group.add_option("-p", "--prec", help='Specify a precision(s) on which to operate.', action='store', dest='precslst', type='string', default="");
284  group.add_option("-e", "--filetypes", help='Specify file extensions on which to operate when walking.', action='store', dest='fileexts', type='string', default="");
285  parser.add_option_group(group);
286 
287  (options, args) = parser.parse_args();
288 
289  """If file extensions are specified, override defaults."""
290  if options.fileexts:
291  EXTS = options.fileexts.split();
292 
293  """Fill the 'work' array with files found to be operable."""
294  if options.fileslst is not None:
295  # Files to examine are specified on the command line
296  # Note fileslst can be "", indicating no files to process
297  for file in options.fileslst.split():
298  check_gen(file, work, rex);
299  else:
300  """Begin directory walking in the current directory."""
301  startDir = '.';
302  for root, dirs, files in os.walk(startDir, True, None):
303  dirs = filter(hidden,dirs);
304  files = filter(hidden,files);
305  files = filter(valid_extension,files);
306  for file in files:
307  check_gen(path.join(root,file), work, rex);
308 
309  """Set static options for conversion."""
310  Conversion.debug = options.debug;
311  Conversion.make = options.make;
312  Conversion.prefix = options.prefix;
313  Conversion.required_precisions = options.precslst.split();
314  if options.out_print or options.out_clean or options.in_print or options.make or options.test:
315  Conversion.test = True;
316 
317  if options.make:
318  """If the program should be GNU Make friendly."""
319  print '## Automatically generated Makefile';
320  print 'PYTHON ?= python';
321 
322  for tuple in work:
323  """For each valid conversion file found."""
324  try:
325  """Try creating and executing a converter."""
326  c = Conversion(tuple[0], tuple[1], tuple[2]);
327  c.run();
328  except Exception, e:
329  print >> sys.stderr, str(e);
330  continue;
331 
332  if options.make:
333  """If the program should be GNU Make friendly."""
334  print 'gen = ',' '+' '.join(Conversion.files_out);
335  print 'cleangen:';
336  print '\trm -f $(gen)';
337  print 'generate: $(gen)';
338  print '.PHONY: cleangen generate';
339  if options.in_print:
340  """Should we print the input files?"""
341  print ' '.join(Conversion.files_in);
342  if options.out_print:
343  """Should we print the output files?"""
344  print ' '.join(Conversion.files_out);
345  if options.out_clean:
346  """Clean generated files"""
347  for file in Conversion.files_out:
348  if not path.exists(file): continue;
349  os.remove(file);
350 
351 if __name__ == "__main__":
352  main();
def relpath
Definition: codegen.py:61
def main
Definition: codegen.py:260
def check_gen
Definition: codegen.py:35
list required_precisions
Definition: codegen.py:79
def grep
Definition: codegen.py:45
def hidden
Definition: codegen.py:49
def valid_extension
Definition: codegen.py:53