Source API extractor

This commit is contained in:
Mark Harrah 2009-11-16 08:46:47 -05:00
parent 74202668c6
commit 2977fd4131
8 changed files with 336 additions and 32 deletions

260
compile/interface/API.scala Normal file
View File

@ -0,0 +1,260 @@
/* sbt -- Simple Build Tool
* Copyright 2008, 2009 Mark Harrah
*/
package xsbt
/*TODO: linearization vs. parents and declared vs. inherited members*/
import java.io.File
import scala.tools.nsc.{io, plugins, symtab, Global, Phase}
import io.{AbstractFile, PlainFile, ZipArchive}
import plugins.{Plugin, PluginComponent}
import symtab.Flags
import scala.collection.mutable.{HashMap, HashSet, ListBuffer}
import xsbti.api.{ClassLike, DefinitionType, PathComponent, SimpleType}
object API
{
val name = "xsbt-api"
}
final class API(val global: Global, val callback: xsbti.AnalysisCallback) extends NotNull
{
import global._
def error(msg: String) = throw new RuntimeException(msg)
def newPhase(prev: Phase) = new ApiPhase(prev)
class ApiPhase(prev: Phase) extends Phase(prev)
{
override def description = "Extracts the public API from source files."
def name = API.name
def run: Unit = currentRun.units.foreach(processUnit)
def processUnit(unit: CompilationUnit)
{
val sourceFile = unit.source.file.file
val traverser = new TopLevelHandler(sourceFile)
traverser.apply(unit.body)
val packages = traverser.packages.toArray[String].map(p => new xsbti.api.Package(p))
val source = new xsbti.api.Source(packages, traverser.definitions.toArray[xsbti.api.Definition])
callback.api(sourceFile, source)
}
}
private def thisPath(sym: Symbol) = path(pathComponents(sym, Constants.thisPath :: Nil))
private def path(components: List[PathComponent]) = new xsbti.api.Path(components.toArray[PathComponent])
private def pathComponents(sym: Symbol, postfix: List[PathComponent]): List[PathComponent] =
{
if(sym == NoSymbol || sym.isRoot || sym.isRootPackage) postfix
else pathComponents(sym.owner, new xsbti.api.Id(sym.simpleName.toString) :: postfix)
}
private def simpleType(t: Type): SimpleType =
processType(t) match
{
case s: SimpleType => s
case _ => error("Expected simple type: " + t)
}
private def types(t: List[Type]): Array[xsbti.api.Type] = t.toArray[Type].map(processType)
private def projectionType(pre: Type, sym: Symbol) =
{
if(pre == NoPrefix) new xsbti.api.ParameterRef(sym.id)
else if(sym.isRoot || sym.isRootPackage) Constants.emptyType
else new xsbti.api.Projection(simpleType(pre), sym.nameString)
}
private def annotations(as: List[AnnotationInfo]): Array[xsbti.api.Annotation] = as.toArray[AnnotationInfo].map(annotation)
private def annotation(a: AnnotationInfo) = new xsbti.api.Annotation(simpleType(a.atp), a.args.map(_.hashCode.toString).toArray[String])
private def annotated(as: List[AnnotationInfo], tpe: Type) = new xsbti.api.Annotated(simpleType(tpe), annotations(as))
private def defDef(s: Symbol) =
{
def build(t: Type, typeParams: Array[xsbti.api.TypeParameter], valueParameters: List[xsbti.api.ParameterList]): xsbti.api.Def =
{
// 2.8 compatibility
implicit def symbolsToParameters(syms: List[Symbol]): xsbti.api.ParameterList =
{
val isImplicitList = syms match { case Nil => false; case head :: _ => isImplicit(head) }
new xsbti.api.ParameterList(syms.map(parameterS).toArray, isImplicitList)
}
// 2.7 compatibility
implicit def typesToParameters(syms: List[Type]): xsbti.api.ParameterList =
{
val isImplicitList = false// TODO: how was this done in 2.7?
new xsbti.api.ParameterList(syms.map(parameterT).toArray, isImplicitList)
}
t match
{
case PolyType(typeParams0, base) =>
assert(typeParams.isEmpty)
assert(valueParameters.isEmpty)
build(base, typeParameters(typeParams0), Nil)
case MethodType(params, resultType) => // in 2.7, params is of type List[Type], in 2.8 it is List[Symbol]
build(resultType, typeParams, (params: xsbti.api.ParameterList) :: valueParameters)
case returnType =>
new xsbti.api.Def(valueParameters.toArray, processType(returnType), typeParams, s.fullNameString, getAccess(s), getModifiers(s))
}
}
def parameterS(s: Symbol): xsbti.api.MethodParameter = makeParameter(s.nameString, s.info, s.info.typeSymbol)
def parameterT(t: Type): xsbti.api.MethodParameter = makeParameter("", t, t.typeSymbol)
def makeParameter(name: String, tpe: Type, ts: Symbol): xsbti.api.MethodParameter =
{
import xsbti.api.ParameterModifier._
val (t, special) =
if(ts == definitions.RepeatedParamClass)// || s == definitions.JavaRepeatedParamClass)
(tpe.typeArgs(0), Repeated)
else if(ts == definitions.ByNameParamClass)
(tpe.typeArgs(0), ByName)
else
(tpe, Plain)
new xsbti.api.MethodParameter(name, processType(t), hasDefault(s), special)
}
build(s.info, Array(), Nil)
}
private def hasDefault(s: Symbol) =
{
// 2.7 compatibility
implicit def flagsWithDefault(f: AnyRef): WithDefault = new WithDefault
class WithDefault { val DEFAULTPARAM = 0x02000000 }
s.hasFlag(Flags.DEFAULTPARAM)
}
private def fieldDef[T](s: Symbol, create: (xsbti.api.Type, String, xsbti.api.Access, xsbti.api.Modifiers) => T): T =
create(processType(s.tpe), s.fullNameString, getAccess(s), getModifiers(s))
private def typeDef(s: Symbol) = error("type members not implemented yet")
private def classStructure(s: Symbol) = structure(s.info.parents, s.info.decls)
private def structure(parents: List[Type], defs: Scope) = new xsbti.api.Structure(types(parents), processDefinitions(defs))
private def processDefinitions(defs: Scope): Array[xsbti.api.Definition] = defs.toList.toArray.map(definition)
private def definition(sym: Symbol): xsbti.api.Definition =
{
if(sym.isClass) classLike(sym)
else if(sym.isMethod) defDef(sym)
else if(sym.isTypeMember) typeDef(sym)
else if(sym.isVariable) fieldDef(sym, new xsbti.api.Var(_,_,_,_))
else fieldDef(sym, new xsbti.api.Val(_,_,_,_))
}
private def getModifiers(s: Symbol): xsbti.api.Modifiers =
{
import Flags._
new xsbti.api.Modifiers(s.hasFlag(ABSTRACT), s.hasFlag(DEFERRED), s.hasFlag(OVERRIDE),
s.isFinal, s.hasFlag(SEALED), isImplicit(s), s.hasFlag(LAZY))
}
private def isImplicit(s: Symbol) = s.hasFlag(Flags.IMPLICIT)
private def getAccess(c: Symbol): xsbti.api.Access =
{
if(c.isPublic) Constants.public
else if(c.isPrivateLocal) Constants.privateLocal
else if(c.isProtectedLocal) Constants.protectedLocal
else
{
val within = c.privateWithin
val qualifier = if(within == NoSymbol) Constants.unqualified else new xsbti.api.IdQualifier(c.fullNameString)
if(c.hasFlag(Flags.PRIVATE)) new xsbti.api.Private(qualifier)
else if(c.hasFlag(Flags.PROTECTED)) new xsbti.api.Protected(qualifier)
else new xsbti.api.Pkg(qualifier)
}
}
private def processType(t: Type): xsbti.api.Type =
{
t match
{
case NoPrefix => Constants.emptyType
case ThisType(sym) => new xsbti.api.Singleton(thisPath(sym))
case SingleType(pre, sym) => projectionType(pre, sym)
case ConstantType(value) => error("Constant type (not implemented)")
case TypeRef(pre, sym, args) =>
val base = projectionType(pre, sym)
if(args.isEmpty) base else new xsbti.api.Parameterized(base, args.map(simpleType).toArray[SimpleType])
case SuperType(thistpe: Type, supertpe: Type) => error("Super type (not implemented)")
case at: AnnotatedType => annotatedType(at)
case RefinedType(parents, defs) => structure(parents, defs)
case ExistentialType(tparams, result) => new xsbti.api.Existential(processType(result), typeParameters(tparams))
case NoType => error("NoType")
case PolyType(typeParams, resultType) => println("polyType(" + typeParams + " , " + resultType + ")"); error("polyType")
case _ => error("Unhandled type " + t.getClass + " : " + t)
}
}
private def annotatedType(at: AnnotatedType): xsbti.api.Type =
{
// In 2.8, attributes is renamed to annotations
implicit def compat(a: AnyRef): WithAnnotations = new WithAnnotations
class WithAnnotations { def attributes = classOf[AnnotatedType].getMethod("annotations").invoke(at).asInstanceOf[List[AnnotationInfo]] }
if(at.attributes.isEmpty) processType(at.underlying) else annotated(at.attributes, at.underlying)
}
private def typeParameters(s: Symbol): Array[xsbti.api.TypeParameter] = typeParameters(s.typeParams)
private def typeParameters(s: List[Symbol]): Array[xsbti.api.TypeParameter] = s.map(typeParameter).toArray[xsbti.api.TypeParameter]
private def typeParameter(s: Symbol): xsbti.api.TypeParameter =
{
val varianceInt = s.variance
import xsbti.api.Variance._
val variance = if(varianceInt < 0) Contravariant else if(varianceInt > 0) Covariant else Invariant
s.info match
{
case TypeBounds(low, high) => new xsbti.api.TypeParameter( s.id, typeParameters(s), variance, processType(low), processType(high) )
case PolyType(typeParams, base) => new xsbti.api.TypeParameter( s.id, typeParameters(typeParams), variance, processType(base.bounds.lo), processType(base.bounds.hi))
case x => error("Unknown type parameter info: " + x.getClass)
}
}
private def selfType(s: Symbol): xsbti.api.Type = if(s.thisSym eq s) Constants.normalSelf else processType(s.typeOfThis)
private def classLike(c: Symbol): ClassLike =
{
val name = c.fullNameString
val access = getAccess(c)
val modifiers = getModifiers(c)
val isModule = c.isModuleClass || c.isModule
val defType =
if(c.isTrait) DefinitionType.Trait
else if(isModule)
{
if(c.isPackage) DefinitionType.PackageModule
else DefinitionType.Module
}
else DefinitionType.ClassDef
new xsbti.api.ClassLike(defType, selfType(c), classStructure(c), typeParameters(c), name, access, modifiers)
}
private final class TopLevelHandler(sourceFile: File) extends TopLevelTraverser
{
val packages = new HashSet[String]
val definitions = new ListBuffer[xsbti.api.Definition]
def `class`(c: Symbol): Unit = definitions += classLike(c)
/** Record packages declared in the source file*/
def `package`(p: Symbol)
{
if( (p eq null) || p == NoSymbol || p.isRoot || p.isRootPackage || p.isEmptyPackageClass || p.isEmptyPackage)
()
else
{
packages += p.fullNameString
`package`(p.enclosingPackage)
}
}
}
private object Constants
{
val public = new xsbti.api.Public
val privateLocal = new xsbti.api.Private(local)
val protectedLocal = new xsbti.api.Protected(local)
val unqualified = new xsbti.api.Unqualified
val local = new xsbti.api.ThisQualifier
val emptyPath = new xsbti.api.Path(Array())
val thisPath = new xsbti.api.This
val emptyType = new xsbti.api.EmptyType
val normalSelf = emptyType
}
private abstract class TopLevelTraverser extends Traverser
{
def `class`(s: Symbol)
def `package`(s: Symbol)
override def traverse(tree: Tree)
{
tree match
{
case (_: ClassDef | _ : ModuleDef) if isTopLevel(tree.symbol) => `class`(tree.symbol)
case p: PackageDef =>
`package`(p.symbol)
super.traverse(tree)
case _ =>
}
}
def isTopLevel(sym: Symbol): Boolean =
(sym ne null) && (sym != NoSymbol) && !sym.isImplClass && !sym.isNestedClass && sym.isStatic &&
!sym.hasFlag(Flags.SYNTHETIC) && !sym.hasFlag(Flags.JAVA)
}
}

View File

@ -23,7 +23,7 @@ final class Analyzer(val global: Global, val callback: AnalysisCallback) extends
def newPhase(prev: Phase): Phase = new AnalyzerPhase(prev)
private class AnalyzerPhase(prev: Phase) extends Phase(prev)
{
override def description = "A plugin to find all concrete instances of a given class and extract dependency information."
override def description = "Extracts dependency information, finds concrete instances of provided superclasses, and application entry points."
def name = Analyzer.name
def run
{

View File

@ -36,11 +36,34 @@ class CompilerInterface
def newPhase(prev: Phase) = analyzer.newPhase(prev)
def name = phaseName
}
object apiExtractor extends
{
val global: compiler.type = compiler
val phaseName = API.name
val runsAfter = List("typer")
override val runsBefore = List("erasure")
val runsRightAfter = Some("typer")
}
with SubComponent with Compat27
{
val api = new API(global, callback)
def newPhase(prev: Phase) = api.newPhase(prev)
def name = phaseName
}
override lazy val phaseDescriptors = // done this way for compatibility between 2.7 and 2.8
{
phasesSet += sbtAnalyzer
phasesSet += apiExtractor
val superd = superComputePhaseDescriptors
if(superd.contains(sbtAnalyzer)) superd else ( superd ++ List(sbtAnalyzer) ).toList
if(superd.contains(sbtAnalyzer))
superd
else
{
val typerIndex = superd.indexOf(analyzer.typerFactory)
assert(typerIndex >= 0)
superd.take(typerIndex+1) ::: apiExtractor :: superd.drop(typerIndex+1) ::: List(sbtAnalyzer)
}
}
private def superComputePhaseDescriptors() = // required because 2.8 makes computePhaseDescriptors private
{

View File

@ -3,16 +3,19 @@ Type
Projection
prefix : SimpleType
id : String
ParameterRef
id: Int
Singleton
path: Path
EmptyType
Parameterized
baseType : SimpleType
typeArguments: SimpleType
typeArguments: SimpleType*
Annotated
baseType : SimpleType
annotations : Annotation*
Structure
parents : Annotated*
parents : Type*
declarations: Definition*
Existential
baseType : Type
@ -23,9 +26,10 @@ Source
definitions: Definition*
Package
components: String*
name: String
Definition
name: String
access: Access
modifiers: Modifiers
FieldLike
@ -38,10 +42,10 @@ Definition
valueParameters: ParameterList*
returnType: Type
ClassLike
definitionType: DefinitionType
selfType: Type
structure: Structure
TypeMember
name: String
TypeAlias
tpe: Type
TypeDeclaration
@ -54,6 +58,7 @@ Access
qualifier: Qualifier
Protected
Private
Pkg
Qualifier
Unqualified
@ -62,6 +67,7 @@ Qualifier
value: String
Modifiers
isAbstract: Boolean
isDeferred: Boolean
isOverride: Boolean
isFinal: Boolean
@ -79,17 +85,19 @@ MethodParameter
modifier: ParameterModifier
TypeParameter
id: Int
typeParameters : TypeParameter*
variance: Variance
upperBound: Type
lowerBound: Type
upperBound: Type
Annotation
base: SimpleType
arguments: String*
enum Variance : Contravariant, Covariant, Invariant
enum ParameterModifier : Vararg, Plain, ByName
enum ParameterModifier : Repeated, Plain, ByName
enum DefinitionType : Trait, ClassDef, Module, PackageModule
Path
components: PathComponent*

View File

@ -34,4 +34,6 @@ public interface AnalysisCallback
public void endSource(File sourcePath);
/** Called when a module with a public 'main' method with the right signature is found.*/
public void foundApplication(File source, String className);
public void api(File sourceFile, xsbti.api.Source source);
}

View File

@ -1,7 +1,7 @@
#Project properties
#Sun Nov 08 13:59:45 EST 2009
#Sat Nov 14 17:25:10 EST 2009
project.organization=org.scala-tools.sbt
project.name=xsbt
sbt.version=0.5.6
project.version=0.6.3
project.version=0.6.4-p1
scala.version=2.7.5

View File

@ -22,14 +22,14 @@ class XSbt(info: ProjectInfo) extends ParentProject(info)
val testSub = project("scripted", "Test", new TestProject(_), ioSub)
val compileAPISub = project(compilePath / "api", "Source API", new CompilerAPIProject(_), datatypeSub)
val compileAPISub = project(compilePath / "api", "Source API", new CompilerAPIProject(_))
val compileInterfaceSub = project(compilePath / "interface", "Compiler Interface", new CompilerInterfaceProject(_), interfaceSub)
val taskSub = project(tasksPath, "Tasks", new TaskProject(_), controlSub, collectionSub)
val cacheSub = project(cachePath, "Cache", new CacheProject(_), taskSub, ioSub)
val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub)
val compilerSub = project(compilePath, "Compile", new CompileProject(_),
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub)
launchInterfaceSub, interfaceSub, ivySub, ioSub, classpathSub, compileInterfaceSub, compileAPISub)
val stdTaskSub = project(tasksPath / "standard", "Standard Tasks", new StandardTaskProject(_), trackingSub, compilerSub)
val distSub = project("dist", "Distribution", new DistProject(_))
@ -134,6 +134,22 @@ class XSbt(info: ProjectInfo) extends ParentProject(info)
val timestamp = formatter.format(new Date)
FileUtilities.write(versionPropertiesPath.asFile, "version=" + version + "\ntimestamp=" + timestamp, log)
}
override def watchPaths = super.watchPaths +++ apiDefinitionPaths --- sources(generatedBasePath)
override def mainSourceRoots = super.mainSourceRoots +++ (generatedBasePath ##)
def srcManagedPath = path("src_managed")
def generatedBasePath = srcManagedPath / "main" / "java"
/** Files that define the datatypes.*/
def apiDefinitionPaths: PathFinder = "definition"
def apiDefinitions = apiDefinitionPaths.get.toList.map(_.absolutePath)
/** Delete up the generated sources*/
lazy val cleanManagedSrc = cleanTask(srcManagedPath)
override def cleanAction = super.cleanAction dependsOn(cleanManagedSrc)
/** Runs the generator compiled by 'compile', putting the classes in src_managed and processing the definitions 'apiDefinitions'. */
lazy val generateSource = generateSourceAction dependsOn(cleanManagedSrc, datatypeSub.compile)
def generateSourceAction = runTask(datatypeSub.getMainClass(true), datatypeSub.runClasspath, "xsbti.api" :: generatedBasePath.absolutePath :: apiDefinitions)
/** compiles the generated sources */
override def compileAction = super.compileAction dependsOn(generateSource)
}
class LaunchInterfaceProject(info: ProjectInfo) extends BaseInterfaceProject(info)
{
@ -146,20 +162,6 @@ class XSbt(info: ProjectInfo) extends ParentProject(info)
/** This subproject generates a hierarchy of Java interfaces and Scala implementations according to a basic format.*/
class CompilerAPIProject(info: ProjectInfo) extends Base(info)
{
override def watchPaths = super.watchPaths +++ apiDefinitionPaths
override def mainSourceRoots = srcManagedPath
def srcManagedPath = ("src_managed" ##)
def generatedBasePath = srcManagedPath / "main" / "java"
/** Files that define the datatypes.*/
def apiDefinitionPaths: PathFinder = "definition"
def apiDefinitions = apiDefinitionPaths.get.toList.map(_.absolutePath)
/** Delete up the generated sources*/
lazy val cleanManagedSrc = cleanTask(srcManagedPath)
/** Runs the generator compiled by 'compile', putting the classes in src_managed and processing the definitions 'apiDefinitions'. */
lazy val generateSource = generateSourceAction dependsOn(cleanManagedSrc)
def generateSourceAction = runTask(datatypeSub.getMainClass(true), datatypeSub.runClasspath, "xsbti.api" :: generatedBasePath.absolutePath :: apiDefinitions)
/** compiles the generated sources */
override def compileAction = super.compileAction dependsOn(generateSource)
}
class CompilerInterfaceProject(info: ProjectInfo) extends Base(info) with SourceProject with TestWithIO with TestWithLog
{

View File

@ -26,38 +26,47 @@ class Generator(pkgName: String, baseDirectory: File)
"}"
writeSource(e.name, content)
}
def write(c: ClassDef): Unit = writeSource(c.name + ".java", classContent(c))
def write(c: ClassDef): Unit = writeSource(c.name, classContent(c))
def classContent(c: ClassDef): String =
{
val allMembers = c.allMembers.map(normalize)
val normalizedMembers = c.members.map(normalize)
val fields = normalizedMembers.map(m => "private final " + m.asJavaDeclaration + ";")
val accessors = normalizedMembers.map(m => "public final " + m.asJavaDeclaration + "() \n\t{\n\t\treturn " + m.name + ";\n\t}")
val parameters = c.allMembers.map(_.asJavaDeclaration)
val parameters = allMembers.map(_.asJavaDeclaration)
val assignments = normalizedMembers.map(m => "this." + m.name + " = " + m.name + ";")
val superConstructor =
{
val inherited = c.inheritedMembers
if(inherited.isEmpty) "" else "super(" + inherited.map(_.name).mkString(", ") + ");"
if(inherited.isEmpty) "" else "super(" + inherited.map(_.name).mkString(", ") + ");\n\t\t"
}
val parametersString = if(allMembers.isEmpty) "\"\"" else allMembers.map(m => fieldToString(m.name, m.single)).mkString(" + \", \" + ")
val toStringMethod = "public String toString()\n\t{\n\t\t" +
"return \"" + c.name + "(\" + " + parametersString + "+ \")\";\n\t" +
"}\n"
val constructor = "public " + c.name + "(" + parameters.mkString(", ") + ")\n\t" +
"{\n\t\t" +
superConstructor + "\n\t\t" +
superConstructor +
assignments.mkString("\n\t\t") + "\n\t" +
"}"
"\nimport java.util.Arrays;\n" +
"public class " + c.name + c.parent.map(" extends " + _.name + " ").getOrElse("") + "\n" +
"{\n\t" +
constructor + "\n\t" +
(fields ++ accessors).mkString("\n\t") + "\n" +
(fields ++ accessors).mkString("\n\t") + "\n\t" +
toStringMethod + "\n" +
"}"
}
def fieldToString(name: String, single: Boolean) = "\"" + name + ": \" + " + fieldString(name + "()", single)
def fieldString(arg: String, single: Boolean) = if(single) arg else "Arrays.toString(" + arg + ")"
def normalize(m: MemberDef): MemberDef =
m.mapType(tpe => if(primitives(tpe.toLowerCase)) tpe.toLowerCase else tpe)
def writeSource(name: String, content: String)
{
import Paths._
val file =baseDirectory / packagePath / name
val file =baseDirectory / packagePath / (name+ ".java")
file.getParentFile.mkdirs()
FileUtilities.write(file, "package " + pkgName + ";\n\n" + content)
}