3 """Build tool that finds dependencies automatically for any language. 5 fabricate is a build tool that finds dependencies automatically for any 6 language. It's small and just works. No hidden stuff behind your back. It was 7 inspired by Bill McCloskey's make replacement, memoize, but fabricate works on 8 Windows as well as Linux. 10 Read more about how to use it and how it works on the project page: 11 https://github.com/SimonAlfie/fabricate/ 13 Like memoize, fabricate is released under a "New BSD license". fabricate is 14 copyright (c) 2009 Brush Technology. Full text of the license is here: 15 https://github.com/SimonAlfie/fabricate/wiki/License 17 To get help on fabricate functions: 18 from fabricate import * 23 from __future__
import with_statement, print_function, unicode_literals
26 __version__ =
'1.29.3' 46 import multiprocessing
50 raise NotImplementedError(
"multiprocessing module not available, can't do parallel builds")
54 PY3 = sys.version_info[0] == 3
57 threading_condition = threading.Condition
59 string_types = basestring
62 threading_condition = threading._Condition
64 threading_condition = threading.Condition
67 __all__ = [
'setup',
'run',
'autoclean',
'main',
'shell',
'fabricate_version',
68 'memoize',
'outofdate',
'parse_options',
'after',
69 'ExecutionError',
'md5_hasher',
'mtime_hasher',
70 'Runner',
'AtimesRunner',
'StraceRunner',
'AlwaysRunner',
71 'SmartRunner',
'Builder']
75 __doc__ +=
"Exported functions are:\n" +
' ' +
'\n '.join(textwrap.wrap(
', '.join(__all__), 80))
79 FAT_atime_resolution = 24*60*60
80 FAT_mtime_resolution = 2
85 NTFS_atime_resolution = 0.0002048
86 NTFS_mtime_resolution = 0.0002048
107 import simplejson
as json
113 return cPickle.load(f)
114 def dump(self, obj, f, indent=None, sort_keys=None):
115 return cPickle.dump(obj, f)
119 """ Print given message to stderr with a line feed. """ 120 print(message, file=sys.stderr)
126 """ Raised by shell() and run() if command returns non-zero exit code. """ 130 """ Return a flat list of the given arguments for shell(). """ 135 if isinstance(arg, (list, tuple)):
138 if not isinstance(arg, string_types):
144 r""" Run a command: program name is given in first arg and command line 145 arguments in the rest of the args. Iterables (lists and tuples) in args 146 are recursively converted to separate arguments, non-string types are 147 converted with str(arg), and None is ignored. For example: 149 >>> def tail(input, n=3, flags=None): 151 >>> return shell('tail', args, flags, input=input) 152 >>> tail('a\nb\nc\nd\ne\n') 154 >>> tail('a\nb\nc\nd\ne\n', 2, ['-v']) 155 '==> standard input <==\nd\ne\n' 157 Keyword arguments kwargs are interpreted as follows: 159 "input" is a string to pass standard input into the process (or the 160 default of None to use parent's stdin, eg: the keyboard) 161 "silent" is True (default) to return process's standard output as a 162 string, or False to print it as it comes out 163 "shell" set to True will run the command via the shell (/bin/sh or 164 COMSPEC) instead of running the command directly (the default) 165 "ignore_status" set to True means ignore command status code -- i.e., 166 don't raise an ExecutionError on nonzero status code 167 Any other kwargs are passed directly to subprocess.Popen 168 Raises ExecutionError(message, output, status) if the command returns 169 a non-zero status code. """ 171 return _shell(args, **kwargs)
176 def _shell(args, input=None, silent=True, shell=False, ignore_status=False, **kwargs):
178 stdin = subprocess.PIPE
182 stdout = subprocess.PIPE
187 raise TypeError(
'shell() takes at least 1 argument (0 given)')
191 command = subprocess.list2cmdline(arglist)
195 proc = subprocess.Popen(command, stdin=stdin, stdout=stdout,
196 stderr=subprocess.STDOUT, shell=shell, **kwargs)
199 if platform.system() ==
'Windows' and e.errno == 2
and e.filename
is None:
200 e.filename = arglist[0]
202 output, stderr = proc.communicate(input)
204 if status
and not ignore_status:
206 % (os.path.basename(arglist[0]), status),
212 """ Return MD5 hash of given filename if it is a regular file or 213 a symlink with a hashable target, or the MD5 hash of the 214 target_filename if it is a symlink without a hashable target, 215 or the MD5 hash of the filename if it is a directory, or None 216 if file doesn't exist. 218 Note: Pyhton versions before 3.2 do not support os.readlink on 219 Windows so symlinks without a hashable target fall back to 220 a hash of the filename if the symlink target is a directory, 221 or None if the symlink is broken""" 222 if not isinstance(filename, bytes):
223 filename = filename.encode(
'utf-8')
225 f =
open(filename,
'rb')
227 return md5func(f.read()).hexdigest()
231 if hasattr(os,
'readlink')
and os.path.islink(filename):
232 return md5func(os.readlink(filename)).hexdigest()
233 elif os.path.isdir(filename):
234 return md5func(filename).hexdigest()
238 """ Return modification time of file, or None if file doesn't exist. """ 240 st = os.stat(filename)
241 return repr(st.st_mtime)
242 except (IOError, OSError):
246 """ Exception raise by Runner constructor if it is not supported 247 on the current platform.""" 252 """ Run command and return (dependencies, outputs), where 253 dependencies is a list of the filenames of files that the 254 command depended on, and output is a list of the filenames 255 of files that the command modified. The input is passed 257 raise NotImplementedError(
"Runner subclass called but subclass didn't define __call__")
260 """ Return the actual runner object (overriden in SmartRunner). """ 264 return self._builder.ignore.search(name)
269 self.
atimes = AtimesRunner.has_atimes(self._builder.dirs)
272 'atimes are not supported on this platform')
276 """ Return whether the given filesystem supports access time updates for 278 - 0 if no a/mtimes not updated 279 - 1 if the atime resolution is at least one day and 280 the mtime resolution at least 2 seconds (as on FAT filesystems) 281 - 2 if the atime and mtime resolutions are both < ms 282 (NTFS filesystem has 100 ns resolution). """ 284 def access_file(filename):
285 """ Access (read a byte from) file to try to update its access time. """ 290 initial = os.stat(filename)
292 initial.st_atime-FAT_atime_resolution,
293 initial.st_mtime-FAT_mtime_resolution))
295 adjusted = os.stat(filename)
296 access_file(filename)
297 after = os.stat(filename)
303 if initial.st_atime-adjusted.st_atime > FAT_atime_resolution+NTFS_atime_resolution
or \
304 initial.st_mtime-adjusted.st_mtime > FAT_mtime_resolution+NTFS_atime_resolution
or \
305 initial.st_atime==adjusted.st_atime
or \
306 initial.st_mtime==adjusted.st_mtime
or \
307 not after.st_atime-FAT_atime_resolution/2 > adjusted.st_atime:
311 initial.st_atime-NTFS_atime_resolution,
312 initial.st_mtime-NTFS_mtime_resolution))
313 adjusted = os.stat(filename)
318 if initial.st_atime-adjusted.st_atime > NTFS_atime_resolution*2
or \
319 initial.st_mtime-adjusted.st_mtime > NTFS_mtime_resolution*2
or \
320 initial.st_atime==adjusted.st_atime
or \
321 initial.st_mtime==adjusted.st_mtime:
328 if not os.path.exists(path):
330 raise PathError(
"build dirs specified a non-existant path '%s'" % path)
334 """ Return whether a file created in each path supports atimes and mtimes. 335 Return value is the same as used by file_has_atimes 336 Note: for speed, this only tests files created at the top directory 337 of each path. A safe assumption in most build environments. 338 In the unusual case that any sub-directories are mounted 339 on alternate file systems that don't support atimes, the build may 340 fail to identify a dependency """ 344 AtimesRunner.exists(path)
345 handle, filename = tempfile.mkstemp(dir=path)
348 f = os.fdopen(handle,
'wb')
356 atimes =
min(atimes, AtimesRunner.file_has_atimes(filename))
362 """ Helper function for file_times(). 363 Return a dict of file times, recursing directories that don't 364 start with self._builder.ignoreprefix """ 366 AtimesRunner.exists(path)
367 names = os.listdir(path)
369 ignoreprefix = self._builder.ignoreprefix
371 if ignoreprefix
and name.startswith(ignoreprefix):
376 fullname = os.path.join(path, name)
377 st = os.stat(fullname)
378 if stat.S_ISDIR(st.st_mode):
381 elif stat.S_ISREG(st.st_mode):
382 times[fullname] = st.st_atime, st.st_mtime
386 """ Return a dict of "filepath: (atime, mtime)" entries for each file 387 in self._builder.dirs. "filepath" is the absolute path, "atime" is 388 the access time, "mtime" the modification time. 389 Recurse directories that don't start with 390 self._builder.ignoreprefix and have depth less than 391 self._builder.dirdepth. """ 394 for path
in self._builder.dirs:
395 times.update(self.
_file_times(path, self._builder.dirdepth))
398 def _utime(self, filename, atime, mtime):
399 """ Call os.utime but ignore permission errors """ 401 os.utime(filename, (atime, mtime))
409 """ Age files' atimes and mtimes to be at least FAT_xx_resolution old. 410 Only adjust if the given filetimes dict says it isn't that old, 411 and return a new dict of filetimes with the ages adjusted. """ 414 for filename, entry
in filetimes.items():
415 if now-entry[0] < FAT_atime_resolution
or now-entry[1] < FAT_mtime_resolution:
416 entry = entry[0] - FAT_atime_resolution, entry[1] - FAT_mtime_resolution
417 self.
_utime(filename, entry[0], entry[1])
418 adjusted[filename] = entry
422 """ Run command and return its dependencies and outputs, using before 423 and after access times to determine dependencies. """ 426 old_stat_float = os.stat_float_times()
427 os.stat_float_times(
True)
436 atime_resolution = FAT_atime_resolution
437 mtime_resolution = FAT_mtime_resolution
438 shell_keywords = dict(silent=
False)
439 shell_keywords.update(kwargs)
440 shell(*args, **shell_keywords)
453 if afters[name][1]-mtime_resolution/2 > befores[name][1]:
456 elif afters[name][0]-atime_resolution/2 > befores[name][0]:
470 for name
in originals:
471 original = originals[name]
472 if original != afters.get(name,
None):
473 self.
_utime(name, original[0], original[1])
475 os.stat_float_times(old_stat_float)
490 self.outputs.add(output)
493 self.delayed_lines.append(line)
496 return '<StraceProcess cwd=%s deps=%s outputs=%s>' % \
500 """ Top level function call for Strace that can be run in parallel """ 501 return self(*args, **kwargs)
512 self.
build_dir = os.path.abspath(build_dir
or os.getcwd())
516 """ Return None if this system doesn't have strace, otherwise 517 return a comma seperated list of system calls supported by strace. """ 518 if platform.system() ==
'Windows':
521 possible_system_calls = [
'open',
'stat',
'stat64',
'lstat',
'lstat64',
522 'execve',
'exit_group',
'chdir',
'mkdir',
'rename',
'clone',
'vfork',
523 'fork',
'symlink',
'creat']
524 valid_system_calls = []
527 for system_call
in possible_system_calls:
528 proc = subprocess.Popen([
'strace',
'-e',
'trace=' + system_call], stderr=subprocess.PIPE)
529 stdout, stderr = proc.communicate()
531 if b
'invalid system call' not in stderr:
532 valid_system_calls.append(system_call)
535 return ','.join(valid_system_calls)
538 _open_re = re.compile(
r'(?P<pid>\d+)\s+open\("(?P<name>[^"]*)", (?P<mode>[^,)]*)')
539 _stat_re = re.compile(
r'(?P<pid>\d+)\s+l?stat(?:64)?\("(?P<name>[^"]*)", .*')
540 _execve_re = re.compile(
r'(?P<pid>\d+)\s+execve\("(?P<name>[^"]*)", .*')
541 _creat_re = re.compile(
r'(?P<pid>\d+)\s+creat\("(?P<name>[^"]*)", .*')
542 _mkdir_re = re.compile(
r'(?P<pid>\d+)\s+mkdir\("(?P<name>[^"]*)", .*\)\s*=\s(?P<result>-?[0-9]*).*')
543 _rename_re = re.compile(
r'(?P<pid>\d+)\s+rename\("[^"]*", "(?P<name>[^"]*)"\)')
544 _symlink_re = re.compile(
r'(?P<pid>\d+)\s+symlink\("[^"]*", "(?P<name>[^"]*)"\)')
545 _kill_re = re.compile(
r'(?P<pid>\d+)\s+killed by.*')
546 _chdir_re = re.compile(
r'(?P<pid>\d+)\s+chdir\("(?P<cwd>[^"]*)"\)')
547 _exit_group_re = re.compile(
r'(?P<pid>\d+)\s+exit_group\((?P<status>.*)\).*')
548 _clone_re = re.compile(
r'(?P<pid_clone>\d+)\s+(clone|fork|vfork)\(.*\)\s*=\s*(?P<pid>\d*)')
553 _unfinished_start_re = re.compile(
r'(?P<pid>\d+)(?P<body>.*)<unfinished ...>$')
554 _unfinished_end_re = re.compile(
r'(?P<pid>\d+)\s+<\.\.\..*>(?P<body>.*)')
557 """ Run strace on given command args/kwargs, sending output to file. 558 Return (status code, list of dependencies, list of outputs). """ 559 shell_keywords = dict(silent=
False)
560 shell_keywords.update(kwargs)
562 shell(
'strace',
'-fo', outname,
'-e',
564 args, **shell_keywords)
565 except ExecutionError
as e:
568 outfile.seek(0, os.SEEK_END)
569 if outfile.tell()
is 0:
584 for pid, process
in processes.items():
585 deps = deps.union(process.deps)
586 outputs = outputs.union(process.outputs)
592 unfinished_start_match = self._unfinished_start_re.match(line)
593 unfinished_end_match = self._unfinished_end_re.match(line)
594 if unfinished_start_match:
595 pid = unfinished_start_match.group(
'pid')
596 body = unfinished_start_match.group(
'body')
597 unfinished[pid] = pid +
' ' + body
599 elif unfinished_end_match:
600 pid = unfinished_end_match.group(
'pid')
601 body = unfinished_end_match.group(
'body')
602 if pid
not in unfinished:
605 printerr(
'fabricate: Warning: resume without unfinished in strace output (strace bug?), \'%s\'' % line.strip())
607 line = unfinished[pid] + body
611 open_match = self._open_re.match(line)
612 stat_match = self._stat_re.match(line)
613 execve_match = self._execve_re.match(line)
614 creat_match = self._creat_re.match(line)
615 mkdir_match = self._mkdir_re.match(line)
616 symlink_match = self._symlink_re.match(line)
617 rename_match = self._rename_re.match(line)
618 clone_match = self._clone_re.match(line)
620 kill_match = self._kill_re.match(line)
622 return None,
None,
None 626 pid = execve_match.group(
'pid')
628 if pid
not in processes
and len(processes) == 0:
632 pid = clone_match.group(
'pid')
633 pid_clone = clone_match.group(
'pid_clone')
634 if pid
not in processes:
639 processes[pid].cwd = processes[pid_clone].cwd
640 processes[pid].delayed =
False 641 for delayed_line
in processes[pid].delayed_lines:
643 self.
_match_line(delayed_line, processes, unfinished)
644 processes[pid].delayed_lines = []
647 mode = match.group(
'mode')
648 if 'O_WRONLY' in mode
or 'O_RDWR' in mode:
659 if match.group(
'result') ==
'0':
663 match = symlink_match
672 name = match.group(
'name')
673 pid = match.group(
'pid')
675 cwd = processes[pid].cwd
677 name = os.path.join(cwd, name)
680 name = os.path.normpath(name)
684 if os.path.isabs(name)
and name.startswith(self.
build_dir):
686 name = name.lstrip(os.path.sep)
688 if (self._builder._is_relevant(name)
690 and os.path.lexists(name)):
692 processes[pid].add_output(name)
694 processes[pid].add_dep(name)
696 match = self._chdir_re.match(line)
698 pid = match.group(
'pid')
700 processes[pid].cwd = os.path.join(processes[pid].cwd, match.group(
'cwd'))
702 match = self._exit_group_re.match(line)
708 if pid
not in processes:
711 process = processes[pid]
713 process.add_delayed_line(line)
719 """ Run command and return its dependencies and outputs, using strace 720 to determine dependencies (by looking at what files are opened or 722 ignore_status = kwargs.pop(
'ignore_status',
False)
726 handle = os.open(outname, os.O_CREAT)
728 handle, outname = tempfile.mkstemp()
732 outfile = os.fdopen(handle,
'r') 737 status, deps, outputs = self.
_do_strace(args, kwargs, outfile, outname)
740 '%r was killed unexpectedly' % args[0],
'', -1)
747 if status
and not ignore_status:
749 % (os.path.basename(args[0]), status),
758 """ Runner that always runs given command, used as a backup in case 759 a system doesn't have strace or atimes. """ 760 shell_keywords = dict(silent=
False)
761 shell_keywords.update(kwargs)
762 shell(*args, **shell_keywords)
766 """ Smart command runner that uses StraceRunner if it can, 767 otherwise AtimesRunner if available, otherwise AlwaysRunner. """ 772 except RunnerUnsupportedException:
775 except RunnerUnsupportedException:
782 return self.
_runner(*args, **kwargs)
785 """ Represents a task put on the parallel pool 786 and its results when complete """ 788 """ "async" is the AsyncResult object returned from pool.apply_async 789 "command" is the command that was run """ 795 """ Represents something waiting on completion of some previous commands """ 797 """ "afters" is a group id or a iterable of group ids to wait on 798 "do" is either a tuple representing a command (group, command, 799 arglist, kwargs) or a threading.Condition to be released """ 805 """ Thread safe mapping object whose values are lists of _running 806 or _after objects and a count of how many have *not* completed """ 808 """ the value type in the map """ 817 self.items.append(val)
825 """ Return copy of the value list """ 827 return self.
groups[id].items[:]
830 """ Remove the group """ 836 self.
groups[id].items.remove(val)
841 self.
groups[id].items.append(val)
844 self.
groups[id].count += 1
847 """if id does not exit, create it without any value""" 856 return self.
groups[id].count
860 c = self.
groups[id].count - 1
876 return self.groups.keys()
883 self.
groups[id].count += 1
884 self.
groups[id].count_in_false += 1
891 self.
groups[id].items.append(val)
894 c = self.
groups[id].count_in_false - 1
897 self.
groups[id].count_in_false = c
907 _stop_results = threading.Event()
910 """ holds the parameters for commands waiting on others """ 911 def __init__(self, group, command, arglist, kwargs):
918 """ Body of thread that stores results in .deps and handles 'after' 920 "builder" the builder used """ 922 while not _stop_results.isSet():
924 for id
in _groups.ids():
925 if id
is False:
continue 926 for r
in _groups.item_list(id):
927 if r.results
is None and r.async.ready():
930 except ExecutionError
as e:
932 _groups.set_ok(id,
False)
933 message, data, status = e
936 builder.done(r.command, d, o)
937 r.results = (r.command, d, o)
938 _groups.dec_count(id)
940 for a
in _groups.item_list(
False):
941 still_to_do =
sum(_groups.get_count(g)
for g
in a.afters)
942 no_error = all(_groups.get_ok(g)
for g
in a.afters)
943 if False in a.afters:
946 if isinstance(a.do, _todo):
948 async = _pool.apply_async(_call_strace, a.do.arglist,
950 _groups.add_for_blocked(a.do.group,
_running(async, a.do.command))
954 _groups.add_for_blocked(a.do.group, r)
956 _groups.set_ok(a.do.group,
False)
957 _groups.dec_count(a.do.group)
958 elif isinstance(a.do, threading_condition):
966 _groups.remove_item(
False, a)
967 _groups.dec_count(
False)
969 _stop_results.wait(delay)
971 etype, eval, etb = sys.exc_info()
972 printerr(
"Error: exception " + repr(etype) +
" at line " +
str(etb.tb_lineno))
973 traceback.print_tb(etb)
975 if not _stop_results.isSet():
979 printerr(
"Error: unexpected results handler exit")
985 You may supply a "runner" class to change the way commands are run 986 or dependencies are determined. For an example, see: 987 https://github.com/SimonAlfie/fabricate/wiki/HowtoMakeYourOwnRunner 989 A "runner" must be a subclass of Runner and must have a __call__() 990 function that takes a command as a list of args and returns a tuple of 991 (deps, outputs), where deps is a list of rel-path'd dependency files 992 and outputs is a list of rel-path'd output files. The default runner 993 is SmartRunner, which automatically picks one of StraceRunner, 994 AtimesRunner, or AlwaysRunner depending on your system. 995 A "runner" class may have an __init__() function that takes the 996 builder as a parameter. 999 def __init__(self, runner=None, dirs=None, dirdepth=100, ignoreprefix='.',
1000 ignore=
None, hasher=md5_hasher, depsname=
'.deps',
1001 quiet=
False, debug=
False, inputs_only=
False, parallel_ok=
False):
1002 """ Initialise a Builder with the given options. 1004 "runner" specifies how programs should be run. It is either a 1005 callable compatible with the Runner class, or a string selecting 1006 one of the standard runners ("atimes_runner", "strace_runner", 1007 "always_runner", or "smart_runner"). 1008 "dirs" is a list of paths to look for dependencies (or outputs) in 1009 if using the strace or atimes runners. 1010 "dirdepth" is the depth to recurse into the paths in "dirs" (default 1011 essentially means infinitely). Set to 1 to just look at the 1012 immediate paths in "dirs" and not recurse at all. This can be 1013 useful to speed up the AtimesRunner if you're building in a large 1014 tree and you don't care about all of the subdirectories. 1015 "ignoreprefix" prevents recursion into directories that start with 1016 prefix. It defaults to '.' to ignore svn directories. 1017 Change it to '_svn' if you use _svn hidden directories. 1018 "ignore" is a regular expression. Any dependency that contains a 1019 regex match is ignored and not put into the dependency list. 1020 Note that the regex may be VERBOSE (spaces are ignored and # line 1021 comments allowed -- use \ prefix to insert these characters) 1022 "hasher" is a function which returns a string which changes when 1023 the contents of its filename argument changes, or None on error. 1024 Default is md5_hasher, but can also be mtime_hasher. 1025 "depsname" is the name of the JSON dependency file to load/save. 1026 "quiet" set to True tells the builder to not display the commands being 1027 executed (or other non-error output). 1028 "debug" set to True makes the builder print debug output, such as why 1029 particular commands are being executed 1030 "inputs_only" set to True makes builder only re-build if input hashes 1031 have changed (ignores output hashes); use with tools that touch 1032 files that shouldn't cause a rebuild; e.g. g++ collect phase 1033 "parallel_ok" set to True to indicate script is safe for parallel running 1052 if runner
is not None:
1054 elif hasattr(self,
'runner'):
1061 is_strace = isinstance(self.runner.actual_runner(), StraceRunner)
1065 _results = threading.Thread(target=_results_handler,
1067 _results.setDaemon(
True)
1070 StraceRunner.keep_temps =
False 1073 """ Print message, but only if builder is not in quiet mode. """ 1078 """ Show a command being executed. Also passed run's "echo" arg 1079 so you can override what's displayed. 1081 if echo
is not None:
1086 """ Show a file being deleted. For subclassing Builder and overriding 1087 this function, the exception is passed in if an OSError occurs 1088 while deleting a file. """ 1090 self.
echo(
'deleting %s' % filename)
1092 self.
echo_debug(
'error deleting %s: %s' % (filename, error.strerror))
1095 """ Print message, but only if builder is in debug mode. """ 1097 print(
'DEBUG: ' + message)
1100 after = kwargs.pop(
'after',
None)
1101 group = kwargs.pop(
'group',
True)
1102 echo = kwargs.pop(
'echo',
None)
1105 raise TypeError(
'run() takes at least 1 argument (0 given)')
1107 command = subprocess.list2cmdline(arglist)
1110 _groups.ensure(group)
1111 return command,
None,
None 1117 _groups.ensure(group)
1118 return command,
None,
None 1123 arglist.insert(0, self.
runner)
1124 if after
is not None:
1125 if not isinstance(after, (list, tuple)):
1130 _groups.inc_count_for_blocked(group)
1135 async = _pool.apply_async(_call_strace, arglist, kwargs)
1136 _groups.add(group,
_running(async, command))
1139 deps, outputs = self.
runner(*arglist, **kwargs)
1140 return self.
done(command, deps, outputs)
1142 def run(self, *args, **kwargs):
1143 """ Run command given in args with kwargs per shell(), but only if its 1144 dependencies or outputs have changed or don't exist. Return tuple 1145 of (command_line, deps_list, outputs_list) so caller or subclass 1148 Parallel operation keyword args "after" specifies a group or 1149 iterable of groups to wait for after they finish, "group" specifies 1150 the group to add this command to. 1152 Optional "echo" keyword arg is passed to echo_command() so you can 1153 override its output if you want. 1156 return self.
_run(*args, **kwargs)
1161 def done(self, command, deps, outputs):
1162 """ Store the results in the .deps file when they are available """ 1163 if deps
is not None or outputs
is not None:
1172 hashed = self.
hasher(dep)
1173 if hashed
is not None:
1174 deps_dict[dep] =
"input-" + hashed
1178 for output
in outputs:
1179 hashed = self.
hasher(output)
1180 if hashed
is not None:
1181 deps_dict[output] =
"output-" + hashed
1186 self.
deps[command] = deps_dict
1188 return command, deps, outputs
1191 """ Run the given command, but only if its dependencies have changed -- 1192 like run(), but returns the status code instead of raising an 1193 exception on error. If "command" is a string (as per memoize.py) 1194 it's split into args using shlex.split() in a POSIX/bash style, 1195 otherwise it's a list of args as per run(). 1197 This function is for compatiblity with memoize.py and is 1198 deprecated. Use run() instead. """ 1199 if isinstance(command, string_types):
1200 args = shlex.split(command)
1204 self.
run(args, **kwargs)
1206 except ExecutionError
as exc:
1207 message, data, status = exc
1211 """ Return True if given build function is out of date. """ 1219 """ Return True if given command line is out of date. """ 1220 if command
in self.
deps:
1222 for dep, oldhash
in self.
deps[command].items():
1223 assert oldhash.startswith(
'input-')
or \
1224 oldhash.startswith(
'output-'), \
1225 "%s file corrupt, do a clean!" % self.
depsname 1226 io_type, oldhash = oldhash.split(
'-', 1)
1235 newhash = self.
hasher(dep)
1236 if newhash
is not None:
1241 self.
echo_debug(
"rebuilding %r, %s %s doesn't exist" %
1242 (command, io_type, dep))
1244 if newhash != oldhash
and (
not self.
inputs_only or io_type ==
'input'):
1245 self.
echo_debug(
"rebuilding %r, hash for %s %s (%s) != old hash (%s)" %
1246 (command, io_type, dep, newhash, oldhash))
1252 self.
echo_debug(
'rebuilding %r, no dependency data' % command)
1258 """ Automatically delete all outputs of this build as well as the .deps 1263 for command, deps
in self.deps.items():
1264 outputs.extend(dep
for dep, hashed
in deps.items()
1265 if hashed.startswith(
'output-'))
1268 for output
in outputs:
1271 except OSError
as e:
1272 if os.path.isdir(output):
1282 for dir
in sorted(dirs, reverse=
True):
1285 except OSError
as e:
1293 """ Lazy load .deps file so that instantiating a Builder is "safe". """ 1294 if not hasattr(self,
'_deps')
or self.
_deps is None:
1300 """ Read dependency JSON file into deps object. """ 1304 self.
_deps = json.load(f)
1306 if self._deps.get(
'.deps_version', 0) != deps_version:
1307 printerr(
'Bad %s dependency file version! Rebuilding.' 1310 self._deps.pop(
'.deps_version',
None)
1317 """ Write out deps object into JSON dependency file. """ 1318 if self.
_deps is None:
1320 self.
deps[
'.deps_version'] = deps_version
1321 if depsname
is None:
1323 f =
open(depsname,
'w')
1325 json.dump(self.
deps, f, indent=4, sort_keys=
True)
1328 self._deps.pop(
'.deps_version',
None)
1331 'atimes_runner' : AtimesRunner,
1332 'strace_runner' : StraceRunner,
1333 'always_runner' : AlwaysRunner,
1334 'smart_runner' : SmartRunner,
1338 """Set the runner for this builder. "runner" is either a Runner 1339 subclass (e.g. SmartRunner), or a string selecting one of the 1340 standard runners ("atimes_runner", "strace_runner", 1341 "always_runner", or "smart_runner").""" 1345 if isinstance(runner, string_types):
1348 self.
runner = getattr(self, runner)
1354 """ Return True if file is in the dependency search directories. """ 1357 fullname = os.path.abspath(fullname)
1358 for path
in self.
dirs:
1359 path = os.path.abspath(path)
1360 if fullname.startswith(path):
1361 rest = fullname[len(path):]
1363 if os.sep+self.
ignoreprefix in os.sep+os.path.dirname(rest):
1366 if rest.count(os.sep) > self.
dirdepth:
1372 """Stops then joins the results handler thread""" 1377 default_builder =
None 1378 default_command =
'build' 1381 _setup_builder =
None 1382 _setup_default =
None 1385 def setup(builder=None, default=None, **kwargs):
1386 """ NOTE: setup functionality is now in main(), setup() is kept for 1387 backward compatibility and should not be used in new scripts. 1389 Setup the default Builder (or an instance of given builder if "builder" 1390 is not None) with the same keyword arguments as for Builder(). 1391 "default" is the name of the default function to run when the build 1392 script is run with no command line arguments. """ 1393 global _setup_builder, _setup_default, _setup_kwargs
1394 _setup_builder = builder
1395 _setup_default = default
1396 _setup_kwargs = kwargs
1397 setup.__doc__ +=
'\n\n' + Builder.__init__.__doc__
1400 """ Set default builder to Builder() instance if it's not yet set. """ 1401 global default_builder
1402 if default_builder
is None:
1406 """ Run the given command, but only if its dependencies have changed. Uses 1407 the default Builder. Return value as per Builder.run(). If there is 1408 only one positional argument which is an iterable treat each element 1409 as a command, returns a list of returns from Builder.run(). 1412 if len(args) == 1
and isinstance(args[0], (list, tuple)):
1413 return [default_builder.run(*a, **kwargs)
for a
in args[0]]
1414 return default_builder.run(*args, **kwargs)
1417 """ wait until after the specified command groups complete and return 1418 results, or None if not parallel """ 1420 if getattr(default_builder,
'parallel_ok',
False):
1422 args = _groups.ids()
1423 cond = threading.Condition()
1426 _groups.add(
False, a)
1433 if a
in ids
and a
is not False:
1435 for i
in _groups.item_list(a):
1437 results.append((a,r))
1443 """ Automatically delete all outputs of the default build. """ 1445 default_builder.autoclean()
1449 return default_builder.memoize(command, **kwargs)
1451 memoize.__doc__ = Builder.memoize.__doc__
1454 """ Return True if given command is out of date and needs to be run. """ 1456 return default_builder.outofdate(command)
1459 _parsed_options =
None 1462 _usage =
'[options] build script functions to run' 1465 """ Parse command line options and return (parser, options, args). """ 1466 parser = optparse.OptionParser(usage=
'Usage: %prog '+usage,
1467 version=
'%prog '+__version__)
1468 parser.disable_interspersed_args()
1469 parser.add_option(
'-t',
'--time', action=
'store_true',
1470 help=
'use file modification times instead of MD5 sums')
1471 parser.add_option(
'-d',
'--dir', action=
'append',
1472 help=
'add DIR to list of relevant directories')
1473 parser.add_option(
'-c',
'--clean', action=
'store_true',
1474 help=
'autoclean build outputs before running')
1475 parser.add_option(
'-q',
'--quiet', action=
'store_true',
1476 help=
"don't echo commands, only print errors")
1477 parser.add_option(
'-D',
'--debug', action=
'store_true',
1478 help=
"show debug info (why commands are rebuilt)")
1479 parser.add_option(
'-k',
'--keep', action=
'store_true',
1480 help=
'keep temporary strace output files')
1481 parser.add_option(
'-j',
'--jobs', type=
'int',
1482 help=
'maximum number of parallel jobs')
1485 for option
in extra_options:
1486 parser.add_option(option)
1487 if command_line
is not None:
1488 options, args = parser.parse_args(command_line)
1490 options, args = parser.parse_args()
1491 global _parsed_options
1492 _parsed_options = (parser, options, args)
1493 return _parsed_options
1496 """ If min is given, assert that the running fabricate is at least that 1497 version or exit with an error message. If max is given, assert that 1498 the running fabricate is at most that version. Return the current 1499 fabricate version string. This function was introduced in v1.14; 1500 for prior versions, the version string is available only as module 1501 local string fabricate.__version__ """ 1503 if min
is not None and float(__version__) < min:
1504 sys.stderr.write((
"fabricate is version %s. This build script " 1505 "requires at least version %.2f") % (__version__, min))
1507 if max
is not None and float(__version__) > max:
1508 sys.stderr.write((
"fabricate is version %s. This build script " 1509 "requires at most version %.2f") % (__version__, max))
1513 def main(globals_dict=None, build_dir=None, extra_options=None, builder=None,
1514 default=
None, jobs=1, command_line=
None, **kwargs):
1515 """ Run the default function or the function(s) named in the command line 1516 arguments. Call this at the end of your build script. If one of the 1517 functions returns nonzero, main will exit with the last nonzero return 1518 value as its status code. 1520 "builder" is the class of builder to create, default (None) is the 1522 "command_line" is an optional list of command line arguments that can 1523 be used to prevent the default parsing of sys.argv. Used to intercept 1524 and modify the command line passed to the build script. 1525 "default" is the default user script function to call, None = 'build' 1526 "extra_options" is an optional list of options created with 1527 optparse.make_option(). The pseudo-global variable main.options 1528 is set to the parsed options list. 1529 "kwargs" is any other keyword arguments to pass to the builder """ 1530 global default_builder, default_command, _pool
1532 kwargs.update(_setup_kwargs)
1533 if _parsed_options
is not None and command_line
is None:
1534 parser, options, actions = _parsed_options
1536 parser, options, actions =
parse_options(extra_options=extra_options, command_line=command_line)
1537 kwargs[
'quiet'] = options.quiet
1538 kwargs[
'debug'] = options.debug
1540 kwargs[
'hasher'] = mtime_hasher
1542 kwargs[
'dirs'] = options.dir
1544 StraceRunner.keep_temps = options.keep
1545 main.options = options
1546 if options.jobs
is not None:
1548 if default
is not None:
1549 default_command = default
1550 if default_command
is None:
1551 default_command = _setup_default
1553 actions = [default_command]
1555 original_path = os.getcwd()
1556 if None in [globals_dict, build_dir]:
1558 frame = sys._getframe(1)
1560 printerr(
"Your Python version doesn't support sys._getframe(1),")
1561 printerr(
"call main(globals(), build_dir) explicitly")
1563 if globals_dict
is None:
1564 globals_dict = frame.f_globals
1565 if build_dir
is None:
1566 build_file = frame.f_globals.get(
'__file__',
None)
1568 build_dir = os.path.dirname(build_file)
1570 if not options.quiet
and os.path.abspath(build_dir) != original_path:
1571 print(
"Entering directory '%s'" % build_dir)
1573 if _pool
is None and jobs > 1:
1574 _pool = multiprocessing.Pool(jobs)
1576 use_builder = Builder
1577 if _setup_builder
is not None:
1578 use_builder = _setup_builder
1579 if builder
is not None:
1580 use_builder = builder
1581 default_builder = use_builder(**kwargs)
1584 default_builder.autoclean()
1588 for action
in actions:
1589 if '(' not in action:
1590 action = action.strip() +
'()' 1591 name = action.split(
'(')[0].
split(
'.')[0]
1592 if name
in globals_dict:
1593 this_status = eval(action, globals_dict)
1595 status =
int(this_status)
1597 printerr(
'%r command not defined!' % action)
1600 except ExecutionError
as exc:
1601 message, data, status = exc.args
1605 if not options.quiet
and os.path.abspath(build_dir) != original_path:
1606 print(
"Leaving directory '%s' back to '%s'" % (build_dir, original_path))
1607 os.chdir(original_path)
1610 if __name__ ==
'__main__':
1616 elif not options.clean:
def _results_handler(builder, delay=0.01)
void split(double tt, double *fr)
def file_has_atimes(filename)
def memoize(self, command, kwargs)
def inc_count_for_blocked(self, id)
def remove_item(self, id, val)
def echo_command(self, command, echo=None)
def done(self, command, deps, outputs)
def __init__(self, runner=None, dirs=None, dirdepth=100, ignoreprefix='.', ignore=None, hasher=md5_hasher, depsname='.deps', quiet=False, debug=False, inputs_only=False, parallel_ok=False)
def _call_strace(self, args, kwargs)
def memoize(command, kwargs)
def __init__(self, cwd='.', delayed=False)
def _run(self, args, kwargs)
def echo_delete(self, filename, error=None)
def cmdline_outofdate(self, command)
def __call__(self, args, kwargs)
def __init__(self, afters, do)
def setup(builder=None, default=None, kwargs)
def _set_default_builder()
def parse_options(usage=_usage, extra_options=None, command_line=None)
def add_delayed_line(self, line)
def __init__(self, async, command)
def __call__(self, args, kwargs)
def __init__(self, builder)
def set_runner(self, runner)
def _age_atimes(self, filetimes)
def __init__(self, builder, build_dir=None)
def mtime_hasher(filename)
def __call__(self, args, kwargs)
def __init__(self, group, command, arglist, kwargs)
def add_for_blocked(self, id, val)
double func(double x, double y)
def _join_results_handler(self)
def write_deps(self, depsname=None)
procfile open("FD_BRL_v0.txt")
static float min(const float a, const float b, const float c)
def _do_strace(self, args, kwargs, outfile, outname)
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
def dump(self, obj, f, indent=None, sort_keys=None)
def fabricate_version(min=None, max=None)
def _is_relevant(self, fullname)
def _matching_is_delayed(self, processes, pid, line)
def echo_debug(self, message)
def _file_times(self, path, depth)
def _utime(self, filename, atime, mtime)
def add_output(self, output)
def __getattr__(self, name)
def __call__(self, args, kwargs)
def outofdate(self, func)
def __init__(self, val=None)
def _shell(args, input=None, silent=True, shell=False, ignore_status=False, kwargs)
def run(self, args, kwargs)
def __call__(self, args, kwargs)
def __init__(self, builder)
def get_strace_system_calls()
def __init__(self, builder)
def main(globals_dict=None, build_dir=None, extra_options=None, builder=None, default=None, jobs=1, command_line=None, kwargs)
def _match_line(self, line, processes, unfinished)