2Code dependency tree analysis and visualization.
4Provides the Tree class for building and navigating compilation and execution
5dependency graphs across multiple FORTRAN source files.
9- Compilation dependency tracking (USE statements)
10- Execution dependency tracking (CALL statements)
11- Source directory scanning
12- Tree serialization to/from JSON
13- Dependency graph visualization (DOT format)
14- Scope reachability analysis
18Tree : Manages cross-file dependency analysis
22>>> tree = Tree(['/path/to/src'], descTreeFile='tree.json')
23>>> deps = tree.needsFile('file.F90')
24>>> tree.plotExecTreeFromScope('module:MOD/sub:SUB', 'deps.png', 2, 2)
25>>> tree.toJson('tree.json') # Save for later use
34from functools
import wraps
41def updateTree(method='file'):
43 Decorator factory to update the tree after PYFTscope method execution.
47 method : str, optional
49 - 'file' (default): Analyze current file.
50 - 'scan': Analyze new files, remove info for deleted files.
51 - 'signal': Analyze files signaled via tree.signal().
53 assert method
in (
'file',
'scan',
'signal')
57 def wrapper(self, *args, **kwargs):
58 result = func(self, *args, **kwargs)
61 self.tree.update(self)
62 elif method ==
'scan':
63 current = set(self.tree.getFiles())
64 old = set(self.tree.knownFiles())
65 self.tree.update(current.symmetric_difference(old))
66 elif method ==
'signal':
67 self.tree.update(self.tree.popSignaled())
75 Build and navigate the code dependency tree.
77 Analyzes FORTRAN source files to build compilation and execution
78 dependency graphs for cross-file analysis.
82 >>> tree = Tree(['/path/to/src'], descTreeFile='tree.json')
83 >>> deps = tree.needsFile('file.F90')
84 >>> tree.plotExecTreeFromScope('module:MOD/sub:SUB', 'deps.png', 2, 2)
86 def __init__(self, tree=None, descTreeFile=None,
87 parserOptions=None, wrapH=False,
90 :param tree: list of directories composing the tree or None
91 :param descTreeFile: filename where the description of the tree will be stored
92 :param parserOptions, wrapH: see the PYFT class
93 :param verbosity: if not None, sets the verbosity level
96 self.
_tree = []
if tree
is None else tree
106 self.
_cwd = os.getcwd()
116 if descTreeFile
is not None and os.path.exists(descTreeFile):
118 elif tree
is not None:
120 if descTreeFile
is not None:
125 :return: a dict containing the full content of the instance
127 return {
'tree': self.
_tree,
145 :param content: Fill the current instance with this dict
147 self.
_tree = content[
'tree']
150 self.
_wrapH = content[
'wrapH']
152 self.
_cwd = content[
'cwd']
153 self.
_scopes = content[
'scopes']
165 Sets self to be a copy of other
171 Sets other to be a copy of self
177 Method used for signaling a modified file which needs to be analized
178 :param filename: file name or PYFTscope object
184 :return: the list of file signaled for update and empties the list
192 :return: the list of analysez file names
194 return list(self.
_scopes.keys())
198 """Is the Tree object valid"""
203 """List of directories"""
209 :param tree: list of directories composing the tree or None
210 :return: list of directories and subdirectories
213 if self.
tree is not None:
214 for tDir
in self.
tree:
215 result += glob.glob(tDir +
'/**/', recursive=
True)
221 :param tree: list of directories composing the tree or None
222 :return: list of directories and subdirectories
225 for tDir
in self.
tree:
226 for filename
in glob.glob(tDir +
'/**/*', recursive=
True):
227 if os.path.splitext(filename)[1]
not in (
'',
'.json',
'.fypp',
231 filenames.append(filename)
237 Builds the self._* variable
246 Updates the object when a file has changed
247 :param file: name of the file (or list of names) with updated content
251 if not isinstance(file, (list, set)):
259 """Empties cached values"""
266 """Fill and return the self._cacheIncInScope cached value"""
274 def _compilationTree(self):
275 """Fill and return the self._cacheCompilationTree cached value"""
280 for filename, incScopePaths
in self.
_includeList.items():
282 for scopePath, incList
in incScopePaths.items():
292 if os.path.normpath(inc) == os.path.normpath(file):
295 elif ((
not os.path.isabs(file))
and
296 os.path.realpath(inc) == os.path.realpath(os.path.join(
297 os.path.dirname(inc), file))):
301 elif os.path.basename(inc) == os.path.basename(file):
303 basename.append(file)
305 same = subdir = basename = []
307 subdir = basename = []
308 if len(basename) > 1:
312 incFilename = same[0]
313 elif len(subdir) > 0:
314 incFilename = subdir[0]
315 elif len(basename) > 0:
316 incFilename = basename[0]
326 for filename, uList
in self.
_useList.items():
328 for modName, _
in [use
for li
in uList.values()
for use
in li]:
329 moduleScopePath =
'module:' + modName
332 for file, scopes
in self.
_scopes.items():
333 if moduleScopePath
in scopes:
338 logging.info(
'Several or none file containing the scope path ' +
339 '%s have been found for file %s',
340 moduleScopePath, filename)
350 def _executionTree(self):
351 """Fill and return the self._cacheCompilationTree cached value"""
355 allScopes = [scopePath
for _, l
in self.
_scopes.items()
for scopePath
in l]
358 for filename, callScopes
in progList.items():
360 for scopePath, cList
in callScopes.items():
362 for call
in set(cList):
367 foundInSameScope = []
370 for kind
in (canonicKind,
'interface'):
372 uList = [self.
_useList[filename][sc]
374 if (sc == scopePath
or scopePath.startswith(sc +
'/'))]
375 for modName, only
in [use
for li
in uList
for use
in li]:
376 moduleScope =
'module:' + modName
377 callScope = moduleScope +
'/' + kind +
':' + call
380 if call
in only
and callScope
in allScopes:
381 foundInUse.append(callScope)
384 for _, scopes
in self.
_scopes.items():
385 if callScope
in scopes:
386 foundInUse.append(callScope)
389 callScope = kind +
':' + call
390 for _, scopes
in self.
_scopes.items():
391 if callScope
in scopes:
392 foundElsewhere.append(callScope)
395 callScope = kind +
':' + call
397 if callScope
in self.
_scopes[incFile]:
398 foundInInclude.append(callScope)
401 callScope = scopePath +
'/' + kind +
':' + call
402 if callScope
in self.
_scopes[filename]:
403 foundInContains.append(callScope)
407 callScope = scopePath.rsplit(
'/', 1)[0] +
'/' + kind + \
410 callScope = kind +
':' + call
411 if callScope
in self.
_scopes[filename]:
412 foundInSameScope.append(callScope)
415 foundInUse = list(set(foundInUse))
416 if len(foundInUse + foundInInclude +
417 foundInContains + foundInSameScope) > 1:
418 logging.error(
'Several definition of the program unit found for '
419 '%s called in %s:', call, scopePath)
420 logging.error(
' found %i time(s) in USE statements',
422 logging.error(
' found %i time(s) in include files',
424 logging.error(
' found %i time(s) in CONTAINS block',
425 len(foundInContains))
426 logging.error(
' found %i time(s) in the same scope',
427 len(foundInSameScope))
429 elif len(foundInUse + foundInInclude +
430 foundInContains + foundInSameScope) == 1:
431 rr = (foundInUse + foundInInclude +
432 foundInContains + foundInSameScope)[0]
433 if canonicKind !=
'func' or rr
in allScopes:
435 elif len(foundElsewhere) > 1:
436 logging.info(
'Several definition of the program unit found for '
437 '%s called in %s', call, scopePath)
438 elif len(foundElsewhere) == 1:
441 if canonicKind !=
'func':
442 logging.info(
'No definition of the program unit found for '
443 '%s called in %s', call, scopePath)
449 for item
in list(execList):
450 itemSplt = item.split(
'/')[-1].split(
':')
451 if itemSplt[0] ==
'interface' and itemSplt[1] !=
'--UNKNOWN--':
453 filenames = [k
for (k, v)
in self.
_scopes.items()
if item
in v]
454 if len(filenames) == 1:
456 execList.remove(item)
457 for sub
in [sub
for sub
in self.
_scopes[filenames[0]]
458 if sub.startswith(item +
'/')]:
459 subscopeIn = sub.rsplit(
'/', 2)[0] +
'/' + sub.split(
'/')[-1]
460 if subscopeIn
in self.
_scopes[filenames[0]]:
462 execList.append(subscopeIn)
464 execList.append(sub.split(
'/')[-1])
475 :param file: Name of the file to explore, or PYFTscope object
476 :return: dict of use, include, call, function and scope list
478 def extractString(text):
480 if text[0]
in (
'"',
"'"):
481 assert text[-1] == text[0]
489 filename = pft.getFileName()
496 filename = filename[2:]
if filename.startswith(
'./')
else filename
504 scopes = pft.getScopes()
507 self.
_scopes[filename].append(scope.path)
509 if scope.path.split(
'/')[-1].split(
':')[0] ==
'interface':
510 for name
in [n2name(nodeN).upper()
511 for moduleproc
in scope.findall(
'./{*}procedure-stmt')
512 for nodeN
in moduleproc.findall(
'./{*}module-procedure-N-LT/' +
515 if re.search(scope.path.rsplit(
'/', 1)[0] +
'/[a-zA-Z]*:' + name,
517 self.
_scopes[filename].append(scope.path +
'/' +
518 sc.path.split(
'/')[-1])
525 [file.text
for file
in scope.findall(
'.//{*}include/{*}filename')]
527 [extractString(file.text)
528 for file
in scope.findall(
'.//{*}include/{*}filename/{*}S')])
532 self.
_useList[filename][scope.path] = []
533 for use
in scope.findall(
'.//{*}use-stmt'):
534 modName = n2name(use.find(
'./{*}module-N/{*}N')).upper()
535 only = [n2name(n).upper()
for n
in use.findall(
'.//{*}use-N//{*}N')]
536 self.
_useList[filename][scope.path].append((modName, only))
541 list(set(n2name(call.find(
'./{*}procedure-designator/{*}named-E/{*}N')).upper()
542 for call
in scope.findall(
'.//{*}call-stmt')))
544 self.
_funcList[filename][scope.path] = set()
545 for name
in [n2name(call.find(
'./{*}N')).upper()
546 for call
in scope.findall(
'.//{*}named-E/{*}R-LT/{*}parens-R/../..')]:
548 var = scope.varList.findVar(name)
549 if var
is None or var[
'as']
is None:
550 self.
_funcList[filename][scope.path].add(name)
563 with open(filename,
'r', encoding=
'utf-8')
as file:
564 descTree = json.load(file)
565 self.
_cwd = descTree[
'cwd']
566 self.
_scopes = descTree[
'scopes']
575 descTree = {
'cwd': self.
_cwd,
583 descTree[
'scopes'] = {k: sorted(descTree[
'scopes'][k])
for k
in sorted(descTree[
'scopes'])}
584 for cat
in (
'useList',
'includeList',
'callList',
'funcList'):
585 descTree[cat] = {file: {scope: sorted(descTree[cat][file][scope])
586 for scope
in sorted(descTree[cat][file])}
587 for file
in sorted(descTree[cat])}
589 with open(filename,
'w', encoding=
'utf-8')
as file:
590 json.dump(descTree, file, indent=2)
595 Return the name of the file defining the scope
596 :param scopePath: scope path to search for
597 :return: list file names in which scope is defined
599 return [filename
for filename, scopes
in self.
_scopes.items()
if scopePath
in scopes]
604 Return the scopes contained in the file
605 :param filename: name of the file tn inspect
606 :return: list of scopes defined in the file
613 :param node: initial node
614 :param descTreePart: 'compilation_tree' or 'execution_tree' part of a descTree object
615 :param level: number of levels (0 to get only the initial node, None to get all nodes)
616 :param down: True to get the nodes lower in the tree, False to get the upper ones
617 :return: list of nodes lower or upper tahn initial node (recursively)
619 def recur(nnn, level, currentList):
621 result = descTreePart.get(nnn, [])
623 result = [item
for (item, l)
in descTreePart.items()
if nnn
in l]
624 if level
is None or level > 1:
625 for res
in list(result):
626 if res
not in currentList:
627 result.extend(recur(res,
None if level
is None else level - 1, result))
629 return recur(node, level, [])
634 :param filename: initial file name
635 :param level: number of levels (0 to get only the initial file, None to get all files)
636 :return: list of file names needed by the initial file (recursively)
643 :param filename: initial file name
644 :param level: number of levels (0 to get only the initial file, None to get all files)
645 :return: list of file names that needs the initial file (recursively)
652 :param scopePath: initial scope path
653 :param level: number of levels (0 to get only the initial scope path,
654 None to get all scopes)
655 :return: list of scopes called by the initial scope path (recursively)
662 :param scopePath: initial scope path
663 :param level: number of levels (0 to get only the initial scope path,
664 None to get all scopes)
665 :return: list of scopes that calls the initial scope path (recursively)
671 includeInterfaces=False, includeStopScopes=False):
673 :param scopePath: scope path to test
674 :param stopScopes: list of scopes
675 :param includeInterfaces: if True, interfaces of positive scopes are also positive
676 :param includeInterfaces: if True, scopes that are in stopScopes return True
677 :return: True if the scope path is called directly or indirectly by one of the scope
678 paths listed in stopScopes
680 scopeSplt = scopePath.split(
'/')
681 if includeInterfaces
and len(scopeSplt) >= 2
and scopeSplt[-2].split(
':')[0] ==
'interface':
684 scopeI = scopeSplt[-1]
688 includeStopScopes=includeStopScopes)
692 return (any(scp
in upperScopes
for scp
in stopScopes)
or
693 (includeStopScopes
and scopePath
in stopScopes))
696 def plotTree(self, centralNodeList, output, plotMaxUpper, plotMaxLower, kind, frame=False):
698 Compute a dependency graph
699 :param centralNodeList: file, scope path, list of files or list of scope paths
700 :param output: output file name (.dot or .png extension)
701 :param plotMaxUpper: Maximum number of elements to plot, upper than the central element
702 :param plotMaxLower: Maximum number of elements to plot, lower than the central element
703 :param kind: must be 'compilation_tree' or 'execution_tree'
704 :param frame: True to plot a frame grouping the central nodes
706 assert kind
in (
'compilation_tree',
'execution_tree')
712 if objHash
not in hashValues:
714 hashValues[objHash] = hashValues[0]
715 return str(hashValues[objHash])
717 def createNode(node, label=None):
719 if label
is not None:
720 result +=
"subgraph cluster_" + myHash(node) +
" {\n"
721 result += f
'label="{label}"\n'
722 if kind ==
'execution_tree':
723 color =
'blue' if node.split(
'/')[-1].split(
':')[0] ==
'func' else 'green'
726 result += myHash(node) + f
' [label="{node}" color="{color}"]\n'
727 if label
is not None:
731 def createLink(file1, file2):
732 return myHash(file1) +
' -> ' + myHash(file2) +
'\n'
734 def createCluster(nodes, label=None):
735 result =
"subgraph cluster_R {\n"
736 result +=
"{rank=same " + (
' '.join([myHash(node)
for node
in nodes])) +
"}\n"
737 if label
is not None:
738 result += f
'label="{label}"\n'
746 def filename(scopePath):
747 if kind ==
'compilation_tree':
749 return [f
for f, l
in self.
_scopes.items()
if scopePath
in l][0]
751 def recur(node, level, down, var):
752 if level
is None or level > 0:
754 result = var.get(node, [])
756 result = [f
for f, l
in var.items()
if node
in l]
758 add(createNode(res, filename(res)))
759 add(createLink(node, res)
if down
else createLink(res, node))
760 if level
is None or level > 1:
761 recur(res,
None if level
is None else level - 1, down, var)
765 if kind ==
'execution_tree':
766 centralScopeFilenames = []
767 for scopePath
in centralNodeList:
768 centralScopeFilenames.append(filename(scopePath))
769 centralScopeFilenames = list(set(centralScopeFilenames))
770 if len(centralScopeFilenames) == 1:
778 var = {k: sorted(var[k])
for k
in sorted(var)}
780 dot = [
"digraph D {\n"]
781 if not isinstance(centralNodeList, list):
782 centralNodeList = [centralNodeList]
783 for centralNode
in centralNodeList:
784 add(createNode(centralNode,
None if printInFrame
else filename(centralNode)))
785 recur(centralNode, plotMaxLower,
True, var)
786 recur(centralNode, plotMaxUpper,
False, var)
788 if kind ==
'compilation_tree':
791 frameText = centralScopeFilenames[0]
if printInFrame
else None
792 add(createCluster(centralNodeList, frameText))
795 fmt = os.path.splitext(output)[1].lower()[1:]
797 with open(output,
'w', encoding=
'utf-8')
as file:
800 dotCommand = [
'dot',
'-T' + fmt,
'-o', output]
801 logging.info(
'Dot command: %s',
' '.join(dotCommand))
802 subprocess.run(dotCommand, input=dot.encode(
'utf8'), check=
True)
807 Compute the compilation dependency graph
808 :param filename: central file
809 :param output: output file name (.dot or .png extension)
810 :param plotMaxUpper: Maximum number of elements to plot, upper than the central element
811 :param plotMaxLower: Maximum number of elements to plot, lower than the central element
813 return self.
plotTree(filename, output, plotMaxUpper, plotMaxLower,
'compilation_tree',
True)
818 Compute the execution dependency graph
819 :param scopePath: central scope path
820 :param output: output file name (.dot or .png extension)
821 :param plotMaxUpper: Maximum number of elements to plot, upper than the central element
822 :param plotMaxLower: Maximum number of elements to plot, lower than the central element
824 return self.
plotTree(scopePath, output, plotMaxUpper, plotMaxLower,
'execution_tree')
829 Compute the compilation dependency graph
830 :param scopePath: central scope path
831 :param output: output file name (.dot or .png extension)
832 :param plotMaxUpper: Maximum number of elements to plot, upper than the central element
833 :param plotMaxLower: Maximum number of elements to plot, lower than the central element
841 Compute the execution dependency graph
842 :param filename: central filename
843 :param output: output file name (.dot or .png extension)
844 :param plotMaxUpper: Maximum number of elements to plot, upper than the central element
845 :param plotMaxLower: Maximum number of elements to plot, lower than the central element
848 'execution_tree',
True)
853 Return the file name containing an interface for the scope path
854 :param scopePath: scope path for which an interface is searched
855 :return: (file name, interface scope) or (None, None) if not found
857 for filename, scopes
in self.
_scopes.items():
858 for scopeInterface
in scopes:
859 if re.search(
r'interface:[a-zA-Z0-9_-]*/' + scopePath, scopeInterface):
860 return filename, scopeInterface