Public Member Functions | Static Public Member Functions | Public Attributes | Static Public Attributes | Private Member Functions | Private Attributes | Static Private Attributes | List of all members
fabricate.StraceRunner Class Reference
Inheritance diagram for fabricate.StraceRunner:
fabricate.Runner object

Public Member Functions

def __init__ (self, builder, build_dir=None)
 
def __call__ (self, args, kwargs)
 
def actual_runner (self)
 
def ignore (self, name)
 

Static Public Member Functions

def get_strace_system_calls ()
 

Public Attributes

 strace_system_calls
 
 temp_count
 
 build_dir
 
 status
 
short int type
 
short int number
 
char name [SIZE_OF_OBJ_NAME]
 
cat_entry star
 

Static Public Attributes

bool keep_temps = False
 

Private Member Functions

def _do_strace (self, args, kwargs, outfile, outname)
 
def _match_line (self, line, processes, unfinished)
 
def _matching_is_delayed (self, processes, pid, line)
 

Private Attributes

 _builder
 

Static Private Attributes

 _open_re = re.compile(r'(?P<pid>\d+)\s+open\("(?P<name>[^"]*)", (?P<mode>[^,)]*)')
 
 _stat_re = re.compile(r'(?P<pid>\d+)\s+l?stat(?:64)?\("(?P<name>[^"]*)", .*')
 
 _execve_re = re.compile(r'(?P<pid>\d+)\s+execve\("(?P<name>[^"]*)", .*')
 
 _creat_re = re.compile(r'(?P<pid>\d+)\s+creat\("(?P<name>[^"]*)", .*')
 
 _mkdir_re = re.compile(r'(?P<pid>\d+)\s+mkdir\("(?P<name>[^"]*)", .*\)\s*=\s(?P<result>-?[0-9]*).*')
 
 _rename_re = re.compile(r'(?P<pid>\d+)\s+rename\("[^"]*", "(?P<name>[^"]*)"\)')
 
 _symlink_re = re.compile(r'(?P<pid>\d+)\s+symlink\("[^"]*", "(?P<name>[^"]*)"\)')
 
 _kill_re = re.compile(r'(?P<pid>\d+)\s+killed by.*')
 
 _chdir_re = re.compile(r'(?P<pid>\d+)\s+chdir\("(?P<cwd>[^"]*)"\)')
 
 _exit_group_re = re.compile(r'(?P<pid>\d+)\s+exit_group\((?P<status>.*)\).*')
 
 _clone_re = re.compile(r'(?P<pid_clone>\d+)\s+(clone|fork|vfork)\(.*\)\s*=\s*(?P<pid>\d*)')
 
 _unfinished_start_re = re.compile(r'(?P<pid>\d+)(?P<body>.*)<unfinished ...>$')
 
 _unfinished_end_re = re.compile(r'(?P<pid>\d+)\s+<\.\.\..*>(?P<body>.*)')
 

Detailed Description

Definition at line 503 of file fabricate.py.

Constructor & Destructor Documentation

def fabricate.StraceRunner.__init__ (   self,
  builder,
  build_dir = None 
)

Definition at line 506 of file fabricate.py.

Referenced by PandAna.core.core.spectrum.fill().

506  def __init__(self, builder, build_dir=None):
507  self.strace_system_calls = StraceRunner.get_strace_system_calls()
508  if self.strace_system_calls is None:
509  raise RunnerUnsupportedException('strace is not available')
510  self._builder = builder
511  self.temp_count = 0
512  self.build_dir = os.path.abspath(build_dir or os.getcwd())
513 
def __init__(self, builder, build_dir=None)
Definition: fabricate.py:506

Member Function Documentation

def fabricate.StraceRunner.__call__ (   self,
  args,
  kwargs 
)
Run command and return its dependencies and outputs, using strace
    to determine dependencies (by looking at what files are opened or
    modified). 

Definition at line 718 of file fabricate.py.

References fabricate.StraceRunner._do_strace(), fabricate.StraceRunner.keep_temps, parse_dependency_file_t.list, and fabricate.StraceRunner.temp_count.

718  def __call__(self, *args, **kwargs):
719  """ Run command and return its dependencies and outputs, using strace
720  to determine dependencies (by looking at what files are opened or
721  modified). """
722  ignore_status = kwargs.pop('ignore_status', False)
723  if self.keep_temps:
724  outname = 'strace%03d.txt' % self.temp_count
725  self.temp_count += 1
726  handle = os.open(outname, os.O_CREAT)
727  else:
728  handle, outname = tempfile.mkstemp()
729 
730  try:
731  try:
732  outfile = os.fdopen(handle, 'r')
733  except:
734  os.close(handle)
735  raise
736  try:
737  status, deps, outputs = self._do_strace(args, kwargs, outfile, outname)
738  if status is None:
739  raise ExecutionError(
740  '%r was killed unexpectedly' % args[0], '', -1)
741  finally:
742  outfile.close()
743  finally:
744  if not self.keep_temps:
745  os.remove(outname)
746 
747  if status and not ignore_status:
748  raise ExecutionError('%r exited with status %d'
749  % (os.path.basename(args[0]), status),
750  '', status)
751  return list(deps), list(outputs)
752 
def __call__(self, args, kwargs)
Definition: fabricate.py:718
def _do_strace(self, args, kwargs, outfile, outname)
Definition: fabricate.py:556
def fabricate.StraceRunner._do_strace (   self,
  args,
  kwargs,
  outfile,
  outname 
)
private
Run strace on given command args/kwargs, sending output to file.
    Return (status code, list of dependencies, list of outputs). 

Definition at line 556 of file fabricate.py.

References fabricate.shell(), and fabricate.StraceRunner.strace_system_calls.

Referenced by fabricate.StraceRunner.__call__().

556  def _do_strace(self, args, kwargs, outfile, outname):
557  """ Run strace on given command args/kwargs, sending output to file.
558  Return (status code, list of dependencies, list of outputs). """
559  shell_keywords = dict(silent=False)
560  shell_keywords.update(kwargs)
561  try:
562  shell('strace', '-fo', outname, '-e',
563  'trace=' + self.strace_system_calls,
564  args, **shell_keywords)
565  except ExecutionError as e:
566  # if strace failed to run, re-throw the exception
567  # we can tell this happend if the file is empty
568  outfile.seek(0, os.SEEK_END)
569  if outfile.tell() is 0:
570  raise e
571  else:
572  # reset the file postion for reading
573  outfile.seek(0)
574 
575  self.status = 0
576  processes = {} # dictionary of processes (key = pid)
577  unfinished = {} # list of interrupted entries in strace log
578  for line in outfile:
579  self._match_line(line, processes, unfinished)
580 
581  # collect outputs and dependencies from all processes
582  deps = set()
583  outputs = set()
584  for pid, process in processes.items():
585  deps = deps.union(process.deps)
586  outputs = outputs.union(process.outputs)
587 
588  return self.status, list(deps), list(outputs)
589 
def shell(args, kwargs)
Definition: fabricate.py:143
def _do_strace(self, args, kwargs, outfile, outname)
Definition: fabricate.py:556
def _match_line(self, line, processes, unfinished)
Definition: fabricate.py:590
def fabricate.StraceRunner._match_line (   self,
  line,
  processes,
  unfinished 
)
private

Definition at line 590 of file fabricate.py.

References fabricate.StraceRunner._match_line(), fabricate.StraceRunner._matching_is_delayed(), fabricate.StraceRunner.build_dir, fabricate.Runner.ignore(), fabricate.Builder.ignore, makeTrainCVSamples.int, fabricate.printerr(), and fabricate.StraceRunner.status.

Referenced by fabricate.StraceRunner._match_line().

590  def _match_line(self, line, processes, unfinished):
591  # look for split lines
592  unfinished_start_match = self._unfinished_start_re.match(line)
593  unfinished_end_match = self._unfinished_end_re.match(line)
594  if unfinished_start_match:
595  pid = unfinished_start_match.group('pid')
596  body = unfinished_start_match.group('body')
597  unfinished[pid] = pid + ' ' + body
598  return
599  elif unfinished_end_match:
600  pid = unfinished_end_match.group('pid')
601  body = unfinished_end_match.group('body')
602  if pid not in unfinished:
603  # Looks like we need to hande an strace bug here
604  # I think it is safe to ignore as I have only seen futex calls which strace should not output
605  printerr('fabricate: Warning: resume without unfinished in strace output (strace bug?), \'%s\'' % line.strip())
606  return
607  line = unfinished[pid] + body
608  del unfinished[pid]
609 
610  is_output = False
611  open_match = self._open_re.match(line)
612  stat_match = self._stat_re.match(line)
613  execve_match = self._execve_re.match(line)
614  creat_match = self._creat_re.match(line)
615  mkdir_match = self._mkdir_re.match(line)
616  symlink_match = self._symlink_re.match(line)
617  rename_match = self._rename_re.match(line)
618  clone_match = self._clone_re.match(line)
619 
620  kill_match = self._kill_re.match(line)
621  if kill_match:
622  return None, None, None
623 
624  match = None
625  if execve_match:
626  pid = execve_match.group('pid')
627  match = execve_match # Executables can be dependencies
628  if pid not in processes and len(processes) == 0:
629  # This is the first process so create dict entry
630  processes[pid] = StraceProcess()
631  elif clone_match:
632  pid = clone_match.group('pid')
633  pid_clone = clone_match.group('pid_clone')
634  if pid not in processes:
635  # Simple case where there are no delayed lines
636  processes[pid] = StraceProcess(processes[pid_clone].cwd)
637  else:
638  # Some line processing was delayed due to an interupted clone_match
639  processes[pid].cwd = processes[pid_clone].cwd # Set the correct cwd
640  processes[pid].delayed = False # Set that matching is no longer delayed
641  for delayed_line in processes[pid].delayed_lines:
642  # Process all the delayed lines
643  self._match_line(delayed_line, processes, unfinished)
644  processes[pid].delayed_lines = [] # Clear the lines
645  elif open_match:
646  match = open_match
647  mode = match.group('mode')
648  if 'O_WRONLY' in mode or 'O_RDWR' in mode:
649  # it's an output file if opened for writing
650  is_output = True
651  elif stat_match:
652  match = stat_match
653  elif creat_match:
654  match = creat_match
655  # a created file is an output file
656  is_output = True
657  elif mkdir_match:
658  match = mkdir_match
659  if match.group('result') == '0':
660  # a created directory is an output file
661  is_output = True
662  elif symlink_match:
663  match = symlink_match
664  # the created symlink is an output file
665  is_output = True
666  elif rename_match:
667  match = rename_match
668  # the destination of a rename is an output file
669  is_output = True
670 
671  if match:
672  name = match.group('name')
673  pid = match.group('pid')
674  if not self._matching_is_delayed(processes, pid, line):
675  cwd = processes[pid].cwd
676  if cwd != '.':
677  name = os.path.join(cwd, name)
678 
679  # normalise path name to ensure files are only listed once
680  name = os.path.normpath(name)
681 
682  # if it's an absolute path name under the build directory,
683  # make it relative to build_dir before saving to .deps file
684  if os.path.isabs(name) and name.startswith(self.build_dir):
685  name = name[len(self.build_dir):]
686  name = name.lstrip(os.path.sep)
687 
688  if (self._builder._is_relevant(name)
689  and not self.ignore(name)
690  and os.path.lexists(name)):
691  if is_output:
692  processes[pid].add_output(name)
693  else:
694  processes[pid].add_dep(name)
695 
696  match = self._chdir_re.match(line)
697  if match:
698  pid = match.group('pid')
699  if not self._matching_is_delayed(processes, pid, line):
700  processes[pid].cwd = os.path.join(processes[pid].cwd, match.group('cwd'))
701 
702  match = self._exit_group_re.match(line)
703  if match:
704  self.status = int(match.group('status'))
705 
def printerr(message)
Definition: fabricate.py:118
def ignore(self, name)
Definition: fabricate.py:263
def _matching_is_delayed(self, processes, pid, line)
Definition: fabricate.py:706
def _match_line(self, line, processes, unfinished)
Definition: fabricate.py:590
def fabricate.StraceRunner._matching_is_delayed (   self,
  processes,
  pid,
  line 
)
private

Definition at line 706 of file fabricate.py.

Referenced by fabricate.StraceRunner._match_line().

706  def _matching_is_delayed(self, processes, pid, line):
707  # Check if matching is delayed and cache a delayed line
708  if pid not in processes:
709  processes[pid] = StraceProcess(delayed=True)
710 
711  process = processes[pid]
712  if process.delayed:
713  process.add_delayed_line(line)
714  return True
715  else:
716  return False
717 
def _matching_is_delayed(self, processes, pid, line)
Definition: fabricate.py:706
def fabricate.Runner.actual_runner (   self)
inherited
Return the actual runner object (overriden in SmartRunner). 

Definition at line 259 of file fabricate.py.

259  def actual_runner(self):
260  """ Return the actual runner object (overriden in SmartRunner). """
261  return self
262 
def actual_runner(self)
Definition: fabricate.py:259
def fabricate.StraceRunner.get_strace_system_calls ( )
static
Return None if this system doesn't have strace, otherwise
    return a comma seperated list of system calls supported by strace. 

Definition at line 515 of file fabricate.py.

516  """ Return None if this system doesn't have strace, otherwise
517  return a comma seperated list of system calls supported by strace. """
518  if platform.system() == 'Windows':
519  # even if windows has strace, it's probably a dodgy cygwin one
520  return None
521  possible_system_calls = ['open','stat', 'stat64', 'lstat', 'lstat64',
522  'execve','exit_group','chdir','mkdir','rename','clone','vfork',
523  'fork','symlink','creat']
524  valid_system_calls = []
525  try:
526  # check strace is installed and if it supports each type of call
527  for system_call in possible_system_calls:
528  proc = subprocess.Popen(['strace', '-e', 'trace=' + system_call], stderr=subprocess.PIPE)
529  stdout, stderr = proc.communicate()
530  proc.wait()
531  if b'invalid system call' not in stderr:
532  valid_system_calls.append(system_call)
533  except OSError:
534  return None
535  return ','.join(valid_system_calls)
536 
def fabricate.Runner.ignore (   self,
  name 
)
inherited

Definition at line 263 of file fabricate.py.

Referenced by fabricate.AtimesRunner.__call__(), and fabricate.StraceRunner._match_line().

263  def ignore(self, name):
264  return self._builder.ignore.search(name)
265 
def ignore(self, name)
Definition: fabricate.py:263

Member Data Documentation

fabricate.StraceRunner._builder
private

Definition at line 510 of file fabricate.py.

fabricate.StraceRunner._chdir_re = re.compile(r'(?P<pid>\d+)\s+chdir\("(?P<cwd>[^"]*)"\)')
staticprivate

Definition at line 546 of file fabricate.py.

fabricate.StraceRunner._clone_re = re.compile(r'(?P<pid_clone>\d+)\s+(clone|fork|vfork)\(.*\)\s*=\s*(?P<pid>\d*)')
staticprivate

Definition at line 548 of file fabricate.py.

fabricate.StraceRunner._creat_re = re.compile(r'(?P<pid>\d+)\s+creat\("(?P<name>[^"]*)", .*')
staticprivate

Definition at line 541 of file fabricate.py.

fabricate.StraceRunner._execve_re = re.compile(r'(?P<pid>\d+)\s+execve\("(?P<name>[^"]*)", .*')
staticprivate

Definition at line 540 of file fabricate.py.

fabricate.StraceRunner._exit_group_re = re.compile(r'(?P<pid>\d+)\s+exit_group\((?P<status>.*)\).*')
staticprivate

Definition at line 547 of file fabricate.py.

fabricate.StraceRunner._kill_re = re.compile(r'(?P<pid>\d+)\s+killed by.*')
staticprivate

Definition at line 545 of file fabricate.py.

fabricate.StraceRunner._mkdir_re = re.compile(r'(?P<pid>\d+)\s+mkdir\("(?P<name>[^"]*)", .*\)\s*=\s(?P<result>-?[0-9]*).*')
staticprivate

Definition at line 542 of file fabricate.py.

fabricate.StraceRunner._open_re = re.compile(r'(?P<pid>\d+)\s+open\("(?P<name>[^"]*)", (?P<mode>[^,)]*)')
staticprivate

Definition at line 538 of file fabricate.py.

fabricate.StraceRunner._rename_re = re.compile(r'(?P<pid>\d+)\s+rename\("[^"]*", "(?P<name>[^"]*)"\)')
staticprivate

Definition at line 543 of file fabricate.py.

fabricate.StraceRunner._stat_re = re.compile(r'(?P<pid>\d+)\s+l?stat(?:64)?\("(?P<name>[^"]*)", .*')
staticprivate

Definition at line 539 of file fabricate.py.

fabricate.StraceRunner._symlink_re = re.compile(r'(?P<pid>\d+)\s+symlink\("[^"]*", "(?P<name>[^"]*)"\)')
staticprivate

Definition at line 544 of file fabricate.py.

fabricate.StraceRunner._unfinished_end_re = re.compile(r'(?P<pid>\d+)\s+<\.\.\..*>(?P<body>.*)')
staticprivate

Definition at line 554 of file fabricate.py.

fabricate.StraceRunner._unfinished_start_re = re.compile(r'(?P<pid>\d+)(?P<body>.*)<unfinished ...>$')
staticprivate

Definition at line 553 of file fabricate.py.

fabricate.StraceRunner.build_dir

Definition at line 512 of file fabricate.py.

Referenced by fabricate.StraceRunner._match_line().

bool fabricate.StraceRunner.keep_temps = False
static

Definition at line 504 of file fabricate.py.

Referenced by fabricate.StraceRunner.__call__().

char object::name
inherited
short int object::number
inherited

Definition at line 115 of file novas.h.

cat_entry object::star
inherited

Definition at line 117 of file novas.h.

fabricate.StraceRunner.status

Definition at line 575 of file fabricate.py.

Referenced by fabricate.StraceRunner._match_line(), and Component.Component.run().

fabricate.StraceRunner.strace_system_calls

Definition at line 507 of file fabricate.py.

Referenced by fabricate.StraceRunner._do_strace().

fabricate.StraceRunner.temp_count

Definition at line 511 of file fabricate.py.

Referenced by fabricate.StraceRunner.__call__().

short int object::type
inherited

Definition at line 114 of file novas.h.


The documentation for this class was generated from the following file: